//**************************************************************************
//**
//**	##   ##    ##    ##   ##   ####     ####   ###     ###
//**	##   ##  ##  ##  ##   ##  ##  ##   ##  ##  ####   ####
//**	 ## ##  ##    ##  ## ##  ##    ## ##    ## ## ## ## ##
//**	 ## ##  ########  ## ##  ##    ## ##    ## ##  ###  ##
//**	  ###   ##    ##   ###    ##  ##   ##  ##  ##       ##
//**	   #    ##    ##    #      ####     ####   ##       ##
//**
//**	$Id: lexer.cpp 2677 2007-08-27 19:02:14Z dj_jl $
//**
//**	Copyright (C) 1999-2006 Jānis Legzdiņš
//**
//**	This program is free software; you can redistribute it and/or
//**  modify it under the terms of the GNU General Public License
//**  as published by the Free Software Foundation; either version 2
//**  of the License, or (at your option) any later version.
//**
//**	This program is distributed in the hope that it will be useful,
//**  but WITHOUT ANY WARRANTY; without even the implied warranty of
//**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//**  GNU General Public License for more details.
//**
//**************************************************************************

// HEADER FILES ------------------------------------------------------------

#include "vcc.h"

// MACROS ------------------------------------------------------------------

// TYPES -------------------------------------------------------------------

// EXTERNAL FUNCTION PROTOTYPES --------------------------------------------

// PUBLIC FUNCTION PROTOTYPES ----------------------------------------------

// PRIVATE FUNCTION PROTOTYPES ---------------------------------------------

// EXTERNAL DATA DECLARATIONS ----------------------------------------------

// PUBLIC DATA DEFINITIONS -------------------------------------------------

TArray<char*>		TLocation::SourceFiles;

const char*			VLexer::TokenNames[] =
{
	"",
	"END OF FILE",
	"IDENTIFIER",
	"NAME LITERAL",
	"STRING LITERAL",
	"INTEGER LITERAL",
	"FLOAT LITERAL",
	//	Keywords
	"abstract",
	"array",
	"bool",
	"break",
	"byte",
	"case",
	"class",
	"const",
	"continue",
	"default",
	"defaultproperties",
	"delegate",
	"do",
	"else",
	"enum",
	"false",
	"final",
	"float",
	"for",
	"foreach",
	"game",
	"get",
	"if",
	"import",
	"int",
	"iterator",
	"name",
	"native",
	"none",
	"NULL",
	"optional",
	"out",
	"private",
	"readonly",
	"reliable",
	"replication",
	"return",
	"self",
	"set",
	"spawner",
	"state",
	"states",
	"static",
	"string",
	"struct",
	"switch",
	"transient",
	"true",
	"unreliable",
	"vector",
	"void",
	"while",
	"__mobjinfo__",
	"__scriptid__",
	//	Punctuation
	"...",
	"<<=",
	">>=",
	"+=",
	"-=",
	"*=",
	"/=",
	"%=",
	"&=",
	"|=",
	"^=",
	"==",
	"!=",
	"<=",
	">=",
	"&&",
	"||",
	"<<",
	">>",
	"++",
	"--",
	"->",
	"::",
	"<",
	">",
	"?",
	"&",
	"|",
	"^",
	"~",
	"!",
	"+",
	"-",
	"*",
	"/",
	"%",
	"(",
	")",
	".",
	",",
	";",
	":",
	"=",
	"[",
	"]",
	"{",
	"}",
};

// PRIVATE DATA DEFINITIONS ------------------------------------------------

// CODE --------------------------------------------------------------------

//==========================================================================
//
//	VLexer::VLexer
//
//==========================================================================

VLexer::VLexer()
: SourceOpen(false)
, Src(NULL)
, Token(TK_NoToken)
, Number(0)
, Float(0)
, Name(NAME_None)
{
	int i;

	memset(TokenStringBuffer, 0, sizeof(TokenStringBuffer));

	for (i = 0; i < 256; i++)
	{
		ASCIIToChrCode[i] = CHR_Special;
		ASCIIToHexDigit[i] = NON_HEX_DIGIT;
	}
	for (i = '0'; i <= '9'; i++)
	{
		ASCIIToChrCode[i] = CHR_Number;
		ASCIIToHexDigit[i] = i-'0';
	}
	for (i = 'A'; i <= 'F'; i++)
	{
		ASCIIToHexDigit[i] = 10+(i-'A');
	}
	for (i = 'a'; i <= 'f'; i++)
	{
		ASCIIToHexDigit[i] = 10+(i-'a');
	}
	for (i = 'A'; i <= 'Z'; i++)
	{
		ASCIIToChrCode[i] = CHR_Letter;
	}
	for (i = 'a'; i <= 'z'; i++)
	{
		ASCIIToChrCode[i] = CHR_Letter;
	}
	ASCIIToChrCode[(int)'\"'] = CHR_Quote;
	ASCIIToChrCode[(int)'\''] = CHR_SingleQuote;
	ASCIIToChrCode[(int)'_'] = CHR_Letter;
	ASCIIToChrCode[0] = CHR_EOF;
	ASCIIToChrCode[EOF_CHARACTER] = CHR_EOF;
	String = TokenStringBuffer;
}

//==========================================================================
//
//	VLexer::OpenSource
//
//==========================================================================

void VLexer::OpenSource(const char* FileName)
{
	//	Read file and prepare for compilation.
	PushSource(Location, FileName);

	SourceOpen = true;

	Token = TK_NoToken;
}

//==========================================================================
//
//	VLexer::~VLexer
//
//==========================================================================

VLexer::~VLexer()
{
	while (Src)
	{
		PopSource();
	}
	SourceOpen = false;
}

//==========================================================================
//
//	VLexer::PushSource
//
//==========================================================================

void VLexer::PushSource(TLocation& Loc, const char* FileName)
{
	FILE* F = fopen(FileName, "rb");
	if (!F)
	{
		FatalError("Couldn't open %s", FileName);
		return;
	}

	VSourceFile* NewSrc = new VSourceFile();
	NewSrc->Next = Src;
	Src = NewSrc;

	//	Copy file name
	NewSrc->FileName = new char[strlen(FileName) + 1];
	strcpy(NewSrc->FileName, FileName);

	//	Extract path to the file.
	const char* PathEnd = FileName + strlen(FileName) - 1;
	while (PathEnd >= FileName && *PathEnd != '/' && *PathEnd != '\\')
	{
		PathEnd--;
	}
	NewSrc->Path = NULL;
	if (PathEnd >= FileName)
	{
		NewSrc->Path = new char[(PathEnd - FileName) + 2];
		strncpy(NewSrc->Path, FileName, (PathEnd - FileName) + 1);
		NewSrc->Path[(PathEnd - FileName) + 1] = 0;
	}

	//	Read the file
	fseek(F, 0, SEEK_END);
	int FileSize = ftell(F);
	fseek(F, 0, SEEK_SET);
	NewSrc->FileStart = new char[FileSize + 1];
	fread(NewSrc->FileStart, 1, FileSize, F);
	fclose(F);
	NewSrc->FileStart[FileSize] = 0;
	NewSrc->FileEnd = NewSrc->FileStart + FileSize;
	NewSrc->FilePtr = NewSrc->FileStart;

	//	Skip garbage some editors add in the begining of UTF-8 files.
	if ((vuint8)NewSrc->FilePtr[0] == 0xef &&
		(vuint8)NewSrc->FilePtr[1] == 0xbb &&
		(vuint8)NewSrc->FilePtr[2] == 0xbf)
	{
		NewSrc->FilePtr += 3;
	}

	//	Save current character and location to be able to restore them.
	NewSrc->Chr = Chr;
	NewSrc->Loc = Location;

	NewSrc->SourceIdx = AddSourceFile(FileName);
	NewSrc->Line = 1;
	NewSrc->IncLineNumber = false;
	NewSrc->NewLine = true;
	NewSrc->Skipping = false;
	Location = TLocation(NewSrc->SourceIdx, NewSrc->Line);
	NextChr();
}

//==========================================================================
//
//	VLexer::PopSource
//
//==========================================================================

void VLexer::PopSource()
{
	if (!Src)
	{
		return;
	}

	if (Src->IfStates.Num())
	{
		ParseError(Location, "#ifdef without a corresponding #endif");
	}

	VSourceFile* Tmp = Src;
	delete[] Tmp->FileName;
	if (Tmp->Path)
	{
		delete[] Tmp->Path;
	}
	delete[] Tmp->FileStart;
	Src = Tmp->Next;
	Chr = Tmp->Chr;
	Location = Tmp->Loc;
	delete Tmp;
}

//==========================================================================
//
//	VLexer::NextToken
//
//==========================================================================

void VLexer::NextToken()
{
	NewLine = Src->NewLine;
	do
	{
		TokenStringBuffer[0] = 0;
		SkipWhitespaceAndComments();
		if (Src->NewLine)
		{
			NewLine = true;
			//	A new line has been started, check preprocessor directive.
			Src->NewLine = false;
			if (Chr == '#')
			{
				ProcessPreprocessor();
				continue;
			}
		}
		switch (ASCIIToChrCode[(vuint8)Chr])
		{
		case CHR_EOF:
			PopSource();
			if (Src)
			{
				Token = TK_NoToken;
			}
			else
			{
				Token = TK_EOF;
			}
			break;
		case CHR_Letter:
			ProcessLetterToken(true);
			break;
		case CHR_Number:
			ProcessNumberToken();
			break;
		case CHR_Quote:
			ProcessQuoteToken();
			break;
		case CHR_SingleQuote:
			ProcessSingleQuoteToken();
			break;
		default:
			ProcessSpecialToken();
			break;
		}
		if (Token != TK_EOF && Src->Skipping)
		{
			Token = TK_NoToken;
		}
	} while (Token == TK_NoToken);
}

//==========================================================================
//
//	VLexer::NextChr
//
//==========================================================================

void VLexer::NextChr()
{
	if (Src->FilePtr >= Src->FileEnd)
	{
		Chr = EOF_CHARACTER;
		return;
	}
	if (Src->IncLineNumber)
	{
		Src->Line++;
		Location = TLocation(Src->SourceIdx, Src->Line);
		Src->IncLineNumber = false;
	}
	Chr = *Src->FilePtr++;
	if ((vuint8)Chr < ' ')
	{
		if (Chr == '\n')
		{
			Src->IncLineNumber = true;
			Src->NewLine = true;
		}
		Chr = ' ';
	}
}

//==========================================================================
//
//	VLexer::SkipWhitespaceAndComments
//
//==========================================================================

void VLexer::SkipWhitespaceAndComments()
{
	bool Done;
	do
	{
		Done = true;
		while (Chr == ' ') NextChr();
		if (Chr == '/' && *Src->FilePtr == '*')
		{
			//	Block comment
			NextChr();
			do
			{
				NextChr();
				if (Chr == EOF_CHARACTER)
				{
					ParseError(Location, "End of file inside a comment");
					return;
				}
			} while (Chr != '*' || *Src->FilePtr != '/');
			NextChr();
			NextChr();
			Done = false;
		}
		else if (Chr == '/' && *Src->FilePtr == '/')
		{
			//	C++ style comment
			NextChr();
			do
			{
				NextChr();
				if (Chr == EOF_CHARACTER)
				{
					ParseError(Location, "End of file inside a comment");
					return;
				}
			} while (!Src->IncLineNumber);
			Done = false;
		}
	} while (!Done);
}

//==========================================================================
//
//	VLexer::ProcessPreprocessor
//
//==========================================================================

void VLexer::ProcessPreprocessor()
{
	NextChr();
	if (Src->NewLine || Chr == EOF_CHARACTER)
	{
		ParseError(Location, "Bad directive.");
		return;
	}
	if (ASCIIToChrCode[(vuint8)Chr] != CHR_Letter)
	{
		ParseError(Location, "Bad directive.");
		while (!Src->NewLine && Chr != EOF_CHARACTER) NextChr();
		return;
	}

	ProcessLetterToken(false);
	if (!strcmp(TokenStringBuffer, "line"))
	{
		//	Read line number
		SkipWhitespaceAndComments();
		if (ASCIIToChrCode[(vuint8)Chr] != CHR_Number)
		{
			ParseError(Location, "Bad directive.");
		}
		ProcessNumberToken();
		Src->Line = Number - 1;

		//	Read file name
		SkipWhitespaceAndComments();
		if (ASCIIToChrCode[(vuint8)Chr] != CHR_Quote)
		{
			ParseError(Location, "Bad directive.");
		}
		ProcessFileName();
		Src->SourceIdx = AddSourceFile(String);
		Location = TLocation(Src->SourceIdx, Src->Line);

		//	Ignore flags
		while (!Src->NewLine)
		{
			NextChr();
		}
	}
	else if (!strcmp(TokenStringBuffer, "define"))
	{
		ProcessDefine();
	}
	else if (!strcmp(TokenStringBuffer, "ifdef"))
	{
		ProcessIf(true);
	}
	else if (!strcmp(TokenStringBuffer, "ifndef"))
	{
		ProcessIf(false);
	}
	else if (!strcmp(TokenStringBuffer, "else"))
	{
		ProcessElse();
	}
	else if (!strcmp(TokenStringBuffer, "endif"))
	{
		ProcessEndIf();
	}
	else if (!strcmp(TokenStringBuffer, "include"))
	{
		ProcessInclude();
		return;
	}
	else
	{
		ParseError(Location, "Bad directive.");
		while (!Src->NewLine && Chr != EOF_CHARACTER) NextChr();
	}
	Token = TK_NoToken;

	SkipWhitespaceAndComments();
	//	A new-line is expected at the end of preprocessor directive.
	if (!Src->NewLine)
	{
		ParseError(Location, "Bad directive.");
	}
}

//==========================================================================
//
//	VLexer::ProcessDefine
//
//==========================================================================

void VLexer::ProcessDefine()
{
	SkipWhitespaceAndComments();
	//	Argument to the #define must be on the same line.
	if (Src->NewLine || Chr == EOF_CHARACTER)
	{
		ParseError(Location, "Bad directive.");
		return;
	}

	//	Parse name to be defined
	if (ASCIIToChrCode[(vuint8)Chr] != CHR_Letter)
	{
		ParseError(Location, "Bad directive.");
		while (!Src->NewLine && Chr != EOF_CHARACTER) NextChr();
		return;
	}
	ProcessLetterToken(false);

	if (Src->Skipping)
	{
		return;
	}

	AddDefine(TokenStringBuffer);
}

//==========================================================================
//
//	VLexer::AddDefine
//
//==========================================================================

void VLexer::AddDefine(const char* CondName)
{
	//	Check for redefined names.
	bool Found = false;
	for (int i = 0; i < Defines.Num(); i++)
	{
		if (!strcmp(Defines[i], CondName))
		{
			ParseWarning(Location, "Redefined conditional");
			Found = true;
			break;
		}
	}
	if (!Found)
	{
		//	Add it.
		char* Copy = new char[strlen(CondName) + 1];
		strcpy(Copy, CondName);
		Defines.Append(Copy);
	}
}

//==========================================================================
//
//	VLexer::ProcessIf
//
//==========================================================================

void VLexer::ProcessIf(bool OnTrue)
{
	SkipWhitespaceAndComments();
	//	Argument to the #ifdef must be on the same line.
	if (Src->NewLine || Chr == EOF_CHARACTER)
	{
		ParseError(Location, "Bad directive.");
		return;
	}

	//	Parse condition name
	if (ASCIIToChrCode[(vuint8)Chr] != CHR_Letter)
	{
		ParseError(Location, "Bad directive.");
		while (!Src->NewLine && Chr != EOF_CHARACTER) NextChr();
		return;
	}
	ProcessLetterToken(false);

	if (Src->Skipping)
	{
		Src->IfStates.Append(IF_Skip);
	}
	else
	{
		//	Check if the names has been defined.
		bool Found = false;
		for (int i = 0; i < Defines.Num(); i++)
		{
			if (!strcmp(Defines[i], TokenStringBuffer))
			{
				Found = true;
				break;
			}
		}
		if (Found == OnTrue)
		{
			Src->IfStates.Append(IF_True);
		}
		else
		{
			Src->IfStates.Append(IF_False);
			Src->Skipping = true;
		}
	}
}

//==========================================================================
//
//	VLexer::ProcessElse
//
//==========================================================================

void VLexer::ProcessElse()
{
	if (!Src->IfStates.Num())
	{
		ParseError(Location, "#else without an #ifdef/#ifndef");
		return;
	}
	switch (Src->IfStates[Src->IfStates.Num() - 1])
	{
	case IF_True:
		Src->IfStates[Src->IfStates.Num() - 1] = IF_ElseFalse;
		Src->Skipping = true;
		break;
	case IF_False:
		Src->IfStates[Src->IfStates.Num() - 1] = IF_ElseTrue;
		Src->Skipping = false;
		break;
	case IF_Skip:
		Src->IfStates[Src->IfStates.Num() - 1] = IF_ElseSkip;
		break;
	case IF_ElseTrue:
	case IF_ElseFalse:
	case IF_ElseSkip:
		ParseError(Location, "Multiple #else directives for a single #ifdef");
		Src->Skipping = true;
		break;
	}
}

//==========================================================================
//
//	VLexer::ProcessEndIf
//
//==========================================================================

void VLexer::ProcessEndIf()
{
	if (!Src->IfStates.Num())
	{
		ParseError(Location, "#endif without an #ifdef/#ifndef");
		return;
	}
	Src->IfStates.RemoveIndex(Src->IfStates.Num() - 1);
	if (Src->IfStates.Num() > 0)
	{
		switch (Src->IfStates[Src->IfStates.Num() - 1])
		{
		case IF_True:
		case IF_ElseTrue:
			Src->Skipping = false;
			break;
		case IF_False:
		case IF_ElseFalse:
			Src->Skipping = true;
			break;
		case IF_Skip:
		case IF_ElseSkip:
			break;
		}
	}
	else
	{
		Src->Skipping = false;
	}
}

//==========================================================================
//
//	VLexer::ProcessInclude
//
//==========================================================================

void VLexer::ProcessInclude()
{
	SkipWhitespaceAndComments();
	//	File name must be on the same line.
	if (Src->NewLine || Chr == EOF_CHARACTER)
	{
		ParseError(Location, "Bad directive.");
		return;
	}

	//	Parse file name
	if (Chr != '\"')
	{
		ParseError(Location, "Bad directive.");
		while (!Src->NewLine && Chr != EOF_CHARACTER) NextChr();
		return;
	}
	ProcessFileName();
	TLocation Loc = Location;

	Token = TK_NoToken;
	SkipWhitespaceAndComments();
	//	A new-line is expected at the end of preprocessor directive.
	if (!Src->NewLine)
	{
		ParseError(Location, "Bad directive.");
	}

	if (Src->Skipping)
	{
		return;
	}

	//	Check if it's an absolute path location.
	if (TokenStringBuffer[0] != '/' && TokenStringBuffer[0] != '\\')
	{
		//	First try relative to the current source file.
		if (Src->Path)
		{
			char* FileName = new char[strlen(Src->Path) +
				strlen(TokenStringBuffer) + 1];
			strcpy(FileName, Src->Path);
			strcat(FileName, TokenStringBuffer);
			FILE* F = fopen(FileName, "r");
			if (F)
			{
				fclose(F);
				PushSource(Loc, FileName);
				delete[] FileName;
				return;
			}
			delete[] FileName;
		}

		for (int i = IncludePath.Num() - 1; i >= 0; i--)
		{
			char* FileName = new char[strlen(IncludePath[i]) +
				strlen(TokenStringBuffer) + 1];
			strcpy(FileName, IncludePath[i]);
			strcat(FileName, TokenStringBuffer);
			FILE* F = fopen(FileName, "r");
			if (F)
			{
				fclose(F);
				PushSource(Loc, FileName);
				delete[] FileName;
				return;
			}
			delete[] FileName;
		}
	}

	//	Either it's relative to the current directory or absolute path.
	PushSource(Loc, TokenStringBuffer);
}

//==========================================================================
//
//	VLexer::AddIncludePath
//
//==========================================================================

void VLexer::AddIncludePath(const char* DirName)
{
	int Len = strlen(DirName);
	char* Copy = new char[Len + 2];
	strcpy(Copy, DirName);
	//	Append trailing slash if needed.
	if (Copy[Len - 1] != '/' && Copy[Len - 1] != '\\')
	{
		Copy[Len] = '/';
		Copy[Len + 1] = 0;
	}
	IncludePath.Append(Copy);
}

//==========================================================================
//
//	VLexer::ProcessNumberToken
//
//==========================================================================

void VLexer::ProcessNumberToken()
{
	char c;

	Token = TK_IntLiteral;
	c = Chr;
	NextChr();
	Number = c - '0';
	if (c == '0' && (Chr == 'x' || Chr == 'X'))
	{
		//  Hexadecimal constant.
		NextChr();
		while (ASCIIToHexDigit[(vuint8)Chr] != NON_HEX_DIGIT)
		{
			Number = (Number << 4) + ASCIIToHexDigit[(vuint8)Chr];
			NextChr();
		}
		return;
	}
	while (ASCIIToChrCode[(vuint8)Chr] == CHR_Number)
	{
		Number = 10 * Number + (Chr - '0');
		NextChr();
	}
	if (Chr == '.')
	{
		Token = TK_FloatLiteral;
		NextChr(); // Point
		Float = Number;
		float	fmul = 0.1;
		while (ASCIIToChrCode[(vuint8)Chr] == CHR_Number)
		{
			Float += (Chr - '0') * fmul;
			fmul /= 10.0;
			NextChr();
		}
		return;
	}
	if (Chr == '_')
	{
		int radix;
		int digit;

		NextChr(); // Underscore
		radix = Number;
		if (radix < 2 || radix > 36)
		{
			ParseError(Location, ERR_BAD_RADIX_CONSTANT);
			radix = 2;
		}
		Number = 0;
		do
		{
			digit = toupper(Chr);
			if (digit < '0' || (digit > '9' && digit < 'A') || digit > 'Z')
			{
				digit = -1;
			}
			else if(digit > '9')
			{
				digit = 10 + digit - 'A';
			}
			else
			{
				digit -= '0';
			}
			if (digit >= radix)
			{
				digit = -1;
			}
			if (digit != -1)
			{
				Number = radix * Number + digit;
				NextChr();
			}
		} while (digit != -1);
	}
}

//==========================================================================
//
//	VLexer::ProcessChar
//
//==========================================================================

void VLexer::ProcessChar()
{
	if (Chr == EOF_CHARACTER)
	{
		ParseError(Location, ERR_EOF_IN_STRING);
		BailOut();
	}
	if (Src->IncLineNumber)
	{
		ParseError(Location, ERR_NEW_LINE_INSIDE_QUOTE);
	}
	if (Chr == '\\')
	{
		//	Special symbol
		NextChr();
		if (Chr == EOF_CHARACTER)
		{
			ParseError(Location, ERR_EOF_IN_STRING);
			BailOut();
		}
		if (Src->IncLineNumber)
		{
			ParseError(Location, ERR_NEW_LINE_INSIDE_QUOTE);
		}
		if (Chr == 'n')
			Chr = '\n';
		else if (Chr == '\'')
			Chr = '\'';
		else if (Chr == '"')
			Chr = '"';
		else if (Chr == 't')
			Chr = '\t';
		else if (Chr == '\\')
			Chr = '\\';
		else
			ParseError(Location, ERR_UNKNOWN_ESC_CHAR);
	}
}

//==========================================================================
//
//	VLexer::ProcessQuoteToken
//
//==========================================================================

void VLexer::ProcessQuoteToken()
{
	int len;

	Token = TK_StringLiteral;
	len = 0;
	NextChr();
	while (Chr != '\"')
	{
		if (len >= MAX_QUOTED_LENGTH - 1)
		{
			ParseError(Location, ERR_STRING_TOO_LONG);
			NextChr();
			continue;
		}
		ProcessChar();
		TokenStringBuffer[len] = Chr;
		NextChr();
		len++;
	}
	TokenStringBuffer[len] = 0;
	NextChr();
}

//==========================================================================
//
//	VLexer::ProcessSingleQuoteToken
//
//==========================================================================

void VLexer::ProcessSingleQuoteToken()
{
	int len;

	Token = TK_NameLiteral;
	len = 0;
	NextChr();
	while (Chr != '\'')
	{
		if (len >= MAX_IDENTIFIER_LENGTH - 1)
		{
			ParseError(Location, ERR_STRING_TOO_LONG);
			NextChr();
			continue;
		}
		ProcessChar();
		TokenStringBuffer[len] = Chr;
		NextChr();
		len++;
	}
	TokenStringBuffer[len] = 0;
	NextChr();
	Name = TokenStringBuffer;
}

//==========================================================================
//
//	VLexer::ProcessLetterToken
//
//==========================================================================

void VLexer::ProcessLetterToken(bool CheckKeywords)
{
	int		len;

	Token = TK_Identifier;
	len = 0;
	while (ASCIIToChrCode[(vuint8)Chr] == CHR_Letter
		|| ASCIIToChrCode[(vuint8)Chr] == CHR_Number)
	{
		if (len == MAX_IDENTIFIER_LENGTH - 1)
		{
			ParseError(Location, ERR_IDENTIFIER_TOO_LONG);
			NextChr();
			continue;
		}
		TokenStringBuffer[len] = Chr;
		len++;
		NextChr();
	}
	TokenStringBuffer[len] = 0;

	if (!CheckKeywords)
	{
		return;
	}

	register const char* s = TokenStringBuffer;
	switch (s[0])
	{
	case '_':
		if (s[1] == '_')
		{
			if (s[2] == 'm' && s[3] == 'o' && s[4] == 'b' && s[5] == 'j' &&
				s[6] == 'i' && s[7] == 'n' && s[8] == 'f' && s[9] == 'o' &&
				s[10] == '_' && s[11] == '_' && s[12] == 0)
			{
				Token = TK_MobjInfo;
			}
			else if (s[2] == 's' && s[3] == 'c' && s[4] == 'r' &&
				s[5] == 'i' && s[6] == 'p' && s[7] == 't' && s[8] == 'i' &&
				s[9] == 'd' && s[10] == '_' && s[11] == '_' && s[12] == 0)
			{
				Token = TK_ScriptId;
			}
		}
		break;

	case 'a':
		if (s[1] == 'b' && s[2] == 's' && s[3] == 't' && s[4] == 'r' &&
			s[5] == 'a' && s[6] == 'c' && s[7] == 't' && s[8] == 0)
		{
			Token = TK_Abstract;
		}
		else if (s[1] == 'r' && s[2] == 'r' && s[3] == 'a' && s[4] == 'y' &&
			s[5] == 0)
		{
			Token = TK_Array;
		}
		break;

	case 'b':
		if (s[1] == 'o' && s[2] == 'o' && s[3] == 'l' && s[4] == 0)
		{
			Token = TK_Bool;
		}
		else if (s[1] == 'r' && s[2] == 'e' && s[3] == 'a' && s[4] == 'k' &&
			s[5] == 0)
		{
			Token = TK_Break;
		}
		else if (s[1] == 'y' && s[2] == 't' && s[3] == 'e' && s[4] == 0)
		{
			Token = TK_Byte;
		}
		break;

	case 'c':
		if (s[1] == 'a' && s[2] == 's' && s[3] == 'e' && s[4] == 0)
		{
			Token = TK_Case;
		}
		else if (s[1] == 'l' && s[2] == 'a' && s[3] == 's' && s[4] == 's' &&
			s[5] == 0)
		{
			Token = TK_Class;
		}
		else if (s[1] == 'o' && s[2] == 'n')
		{
			if (s[3] == 's' && s[4] == 't' && s[5] == 0)
			{
				Token = TK_Const;
			}
			else if (s[3] == 't' && s[4] == 'i' && s[5] == 'n' &&
				s[6] == 'u' && s[7] == 'e' && s[8] == 0)
			{
				Token = TK_Continue;
			}
		}
		break;

	case 'd':
		if (s[1] == 'e')
		{
			if (s[2] == 'f' && s[3] == 'a' && s[4] == 'u' && s[5] == 'l' &&
				s[6] == 't')
			{
				if (s[7] == 0)
				{
					Token = TK_Default;
				}
				else if (s[7] == 'p' && s[8] == 'r' && s[9] == 'o' &&
					s[10] == 'p' && s[11] == 'e' && s[12] == 'r' &&
					s[13] == 't' && s[14] == 'i' && s[15] == 'e' &&
					s[16] == 's' && s[17] == 0)
				{
					Token = TK_DefaultProperties;
				}
			}
			else if (s[2] == 'l' && s[3] == 'e' && s[4] == 'g' &&
				s[5] == 'a' && s[6] == 't' && s[7] == 'e' && s[8] == 0)
			{
				Token = TK_Delegate;
			}
		}
		else if (s[1] == 'o' && s[2] == 0)
		{
			Token = TK_Do;
		}
		break;

	case 'e':
		if (s[1] == 'l' && s[2] == 's' && s[3] == 'e' && s[4] == 0)
		{
			Token = TK_Else;
		}
		else if (s[1] == 'n' && s[2] == 'u' && s[3] == 'm' && s[4] == 0)
		{
			Token = TK_Enum;
		}
		break;

	case 'f':
		if (s[1] == 'a' && s[2] == 'l' && s[3] == 's' && s[4] == 'e' &&
			s[5] == 0)
		{
			Token = TK_False;
		}
		else if (s[1] == 'i' && s[2] == 'n' && s[3] == 'a' && s[4] == 'l' &&
			s[5] == 0)
		{
			Token = TK_Final;
		}
		else if (s[1] == 'l' && s[2] == 'o' && s[3] == 'a' && s[4] == 't' &&
			s[5] == 0)
		{
			Token = TK_Float;
		}
		else if (s[1] == 'o' && s[2] == 'r')
		{
			if (s[3] == 0)
			{
				Token = TK_For;
			}
			else if (s[3] == 'e' && s[4] == 'a' && s[5] == 'c' &&
				s[6] == 'h' && s[7] == 0)
			{
				Token = TK_Foreach;
			}
		}
		break;

	case 'g':
		if (s[1] == 'a' && s[2] == 'm' && s[3] == 'e' && s[4] == 0)
		{
			Token = TK_Game;
		}
		else if (s[1] == 'e' && s[2] == 't' && s[3] == 0)
		{
			Token = TK_Get;
		}

	case 'i':
		if (s[1] == 'f' && s[2] == 0)
		{
			Token = TK_If;
		}
		else if (s[1] == 'm' && s[2] == 'p' && s[3] == 'o' && s[4] == 'r' &&
			s[5] == 't' && s[6] == 0)
		{
			Token = TK_Import;
		}
		else if (s[1] == 'n' && s[2] == 't' && s[3] == 0)
		{
			Token = TK_Int;
		}
		else if (s[1] == 't' && s[2] == 'e' && s[3] == 'r' && s[4] == 'a' &&
			s[5] == 't' && s[6] == 'o' && s[7] == 'r' && s[8] == 0)
		{
			Token = TK_Iterator;
		}
		break;

	case 'n':
		if (s[1] == 'a')
		{
			if (s[2] == 'm' && s[3] == 'e' && s[4] == 0)
			{
				Token = TK_Name;
			}
			if (s[2] == 't' && s[3] == 'i' && s[4] == 'v' && s[5] == 'e' &&
				s[6] == 0)
			{
				Token = TK_Native;
			}
		}
		else if (s[1] == 'o' && s[2] == 'n' && s[3] == 'e' && s[4] == 0)
		{
			Token = TK_None;
		}
		break;

/*	case 'N':
		if (s[1] == 'U' && s[2] == 'L' &&
			s[3] == 'L' && s[4] == 0)
		{
			Token = TK_KEYWORD;
			tk_Keyword = KW_NULL;
		}
		break;*/

	case 'o':
		if (s[1] == 'p' && s[2] == 't' && s[3] == 'i' && s[4] == 'o' &&
			s[5] == 'n' && s[6] == 'a' && s[7] == 'l' && s[8] == 0)
		{
			Token = TK_Optional;
		}
		else if (s[1] == 'u' && s[2] == 't' && s[3] == 0)
		{
			Token = TK_Out;
		}
		break;

	case 'p':
		if (s[1] == 'r' && s[2] == 'i' && s[3] == 'v' && s[4] == 'a' &&
			s[5] == 't' && s[6] == 'e' && s[7] == 0)
		{
			Token = TK_Private;
		}
		break;

	case 'r':
		if (s[1] == 'e')
		{
			if (s[2] == 'a' && s[3] == 'd' && s[4] == 'o' && s[5] == 'n' &&
				s[6] == 'l' && s[7] == 'y' && s[8] == 0)
			{
				Token = TK_ReadOnly;
			}
			else if (s[2] == 'l' && s[3] == 'i' && s[4] == 'a' &&
				s[5] == 'b' && s[6] == 'l' && s[7] == 'e' && s[8] == 0)
			{
				Token = TK_Reliable;
			}
			else if (s[2] == 'p' && s[3] == 'l' && s[4] == 'i' &&
				s[5] == 'c' && s[6] == 'a' && s[7] == 't' && s[8] == 'i' &&
				s[9] == 'o' && s[10] == 'n' && s[11] == 0)
			{
				Token = TK_Replication;
			}
			else if (s[2] == 't' && s[3] == 'u' && s[4] == 'r' &&
				s[5] == 'n' && s[6] == 0)
			{
				Token = TK_Return;
			}
		}
		break;

	case 's':
		if (s[1] == 'e')
		{
			if (s[2] == 'l' && s[3] == 'f' && s[4] == 0)
			{
				Token = TK_Self;
			}
			else if (s[2] == 't' && s[3] == 0)
			{
				Token = TK_Set;
			}
		}
		else if (s[1] == 'p' && s[2] == 'a' && s[3] == 'w' && s[4] == 'n' &&
			s[5] == 'e' && s[6] == 'r' && s[7] == 0)
		{
			Token = TK_Spawner;
		}
		else if (s[1] == 't')
		{
			if (s[2] == 'a' && s[3] == 't')
			{
				if (s[4] == 'e')
				{
					if (s[5] == 0)
					{
						Token = TK_State;
					}
					else if (s[5] == 's' && s[6] == 0)
					{
						Token = TK_States;
					}
				}
				else if (s[4] == 'i' && s[5] == 'c' && s[6] == 0)
				{
					Token = TK_Static;
				}
			}
			else if (s[2] == 'r')
			{
				if (s[3] == 'i' && s[4] == 'n' && s[5] == 'g' && s[6] == 0)
				{
					Token = TK_String;
				}
				else if (s[3] == 'u' && s[4] == 'c' && s[5] == 't' &&
					s[6] == 0)
				{
					Token = TK_Struct;
				}
			}
		}
		else if (s[1] == 'w' && s[2] == 'i' && s[3] == 't' && s[4] == 'c' &&
			s[5] == 'h' && s[6] == 0)
		{
			Token = TK_Switch;
		}
		break;

	case 't':
		if (s[1] == 'r')
		{
			if (s[2] == 'a' && s[3] == 'n' && s[4] == 's' && s[5] == 'i' &&
				s[6] == 'e' && s[7] == 'n' && s[8] == 't' && s[9] == 0)
			{
				Token = TK_Transient;
			}
			else if (s[2] == 'u' && s[3] == 'e' && s[4] == 0)
			{
				Token = TK_True;
			}
		}
		break;

	case 'u':
		if (s[1] == 'n' && s[2] == 'r' && s[3] == 'e' && s[4] == 'l' &&
			s[5] == 'i' && s[6] == 'a' && s[7] == 'b' && s[8] == 'l' &&
			s[9] == 'e' && s[10] == 0)
		{
			Token = TK_Unreliable;
		}
		break;

	case 'v':
		if (s[1] == 'e' && s[2] == 'c' && s[3] == 't' && s[4] == 'o' &&
			s[5] == 'r' && s[6] == 0)
		{
			Token = TK_Vector;
		}
		else if (s[1] == 'o' && s[2] == 'i' && s[3] == 'd' && s[4] == 0)
		{
			Token = TK_Void;
		}
		break;

	case 'w':
		if (s[1] == 'h' && s[2] == 'i' && s[3] == 'l' && s[4] == 'e' &&
			s[5] == 0)
		{
			Token = TK_While;
		}
		break;
	}
	if (s[0] == 'N' && s[1] == 'U' && s[2] == 'L' && s[3] == 'L' && s[4] == 0)
	{
		Token = TK_Null;
	}

	if (Token == TK_Identifier)
	{
		Name = TokenStringBuffer;
	}
}

//==========================================================================
//
//	VLexer::ProcessSpecialToken
//
//==========================================================================

void VLexer::ProcessSpecialToken()
{
	char c = Chr;
	NextChr();
	switch (c)
	{
	case '+':
		if (Chr == '=')
		{
			Token = TK_AddAssign;
			NextChr();
		}
		else if (Chr == '+')
		{
			Token = TK_Inc;
			NextChr();
		}
		else
		{
			Token = TK_Plus;
		}
		break;

	case '-':
		if (Chr == '=')
		{
			Token = TK_MinusAssign;
			NextChr();
		}
		else if (Chr == '-')
		{
			Token = TK_Dec;
			NextChr();
		}
		else if (Chr == '>')
		{
			Token = TK_Arrow;
			NextChr();
		}
		else
		{
			Token = TK_Minus;
		}
		break;

	case '*':
		if (Chr == '=')
		{
			Token = TK_MultiplyAssign;
			NextChr();
		}
		else
		{
			Token = TK_Asterisk;
		}
		break;

	case '/':
		if (Chr == '=')
		{
			Token = TK_DivideAssign;
			NextChr();
		}
		else
		{
			Token = TK_Slash;
		}
		break;

	case '%':
		if (Chr == '=')
		{
			Token = TK_ModAssign;
			NextChr();
		}
		else
		{
			Token = TK_Percent;
		}
		break;

	case '=':
		if (Chr == '=')
		{
			Token = TK_Equals;
			NextChr();
		}
		else
		{
			Token = TK_Assign;
		}
		break;

	case '<':
		if (Chr == '<')
		{
			NextChr();
			if (Chr == '=')
			{
				Token = TK_LShiftAssign;
				NextChr();
			}
			else
			{
				Token = TK_LShift;
			}
		}
		else if (Chr == '=')
		{
			Token = TK_LessEquals;
			NextChr();
		}
		else
		{
			Token = TK_Less;
		}
		break;

	case '>':
		if (Chr == '>')
		{
			NextChr();
			if (Chr == '=')
			{
				Token = TK_RShiftAssign;
				NextChr();
			}
			else
			{
				Token = TK_RShift;
			}
		}
		else if (Chr == '=')
		{
			Token = TK_GreaterEquals;
			NextChr();
		}
		else
		{
			Token = TK_Greater;
		}
		break;

	case '!':
		if (Chr == '=')
		{
			Token = TK_NotEquals;
			NextChr();
		}
		else
		{
			Token = TK_Not;
		}
		break;

	case '&':
		if (Chr == '=')
		{
			Token = TK_AndAssign;
			NextChr();
		}
		else if (Chr == '&')
		{
			Token = TK_AndLog;
			NextChr();
		}
		else
		{
			Token = TK_And;
		}
		break;

	case '|':
		if (Chr == '=')
		{
			Token = TK_OrAssign;
			NextChr();
		}
		else if (Chr == '|')
		{
			Token = TK_OrLog;
			NextChr();
		}
		else
		{
			Token = TK_Or;
		}
		break;

	case '^':
		if (Chr == '=')
		{
			Token = TK_XOrAssign;
			NextChr();
		}
		else
		{
			Token = TK_XOr;
		}
		break;

	case '.':
		if (Chr == '.' && Src->FilePtr[0] == '.')
		{
			Token = TK_VarArgs;
			NextChr();
			NextChr();
		}
		else
		{
			Token = TK_Dot;
		}
		break;

	case ':':
		if (Chr == ':')
		{
			Token = TK_DColon;
			NextChr();
		}
		else
		{
			Token = TK_Colon;
		}
		break;

	case '(':
		Token = TK_LParen;
		break;

	case ')':
		Token = TK_RParen;
		break;

	case '?':
		Token = TK_Quest;
		break;

	case '~':
		Token = TK_Tilde;
		break;

	case ',':
		Token = TK_Comma;
		break;

	case ';':
		Token = TK_Semicolon;
		break;

	case '[':
		Token = TK_LBracket;
		break;

	case ']':
		Token = TK_RBracket;
		break;

	case '{':
		Token = TK_LBrace;
		break;

	case '}':
		Token = TK_RBrace;
		break;

	default:
		ParseError(Location, ERR_BAD_CHARACTER, "Unknown punctuation \'%c\'", Chr);
		Token = TK_NoToken;
	}
}

//==========================================================================
//
//	VLexer::ProcessFileName
//
//==========================================================================

void VLexer::ProcessFileName()
{
	int len = 0;
	NextChr();
	while (Chr != '\"')
	{
		if (len >= MAX_QUOTED_LENGTH - 1)
		{
			ParseError(Location, ERR_STRING_TOO_LONG);
			NextChr();
			continue;
		}
		if (Chr == EOF_CHARACTER)
		{
			ParseError(Location, ERR_EOF_IN_STRING);
			break;
		}
		if (Src->IncLineNumber)
		{
			ParseError(Location, ERR_NEW_LINE_INSIDE_QUOTE);
		}
		TokenStringBuffer[len] = Chr;
		NextChr();
		len++;
	}
	TokenStringBuffer[len] = 0;
	NextChr();
}

//==========================================================================
//
//	VLexer::AddSourceFile
//
//==========================================================================

int VLexer::AddSourceFile(const char* SName)
{
	//	Find it.
	for (int i = 0; i < TLocation::SourceFiles.Num(); i++)
		if (!strcmp(SName, TLocation::SourceFiles[i]))
			return i;

	//	Not found, add it.
	char* NewName = new char[strlen(SName) + 1];
	strcpy(NewName, SName);
	return TLocation::SourceFiles.Append(NewName);
}

//==========================================================================
//
//	VLexer::Check
//
//==========================================================================

bool VLexer::Check(EToken tk)
{
	if (Token == tk)
	{
		NextToken();
		return true;
	}
	return false;
}

//==========================================================================
//
//	VLexer::Expect
//
//	Report error, if current token is not equals to tk.
//	Take next token.
//
//==========================================================================

void VLexer::Expect(EToken tk)
{
	if (Token != tk)
	{
		ParseError(Location, "expected %s, found %s", TokenNames[tk],
			TokenNames[Token]);
	}
	NextToken();
}

//==========================================================================
//
//	VLexer::Expect
//
//	Report error, if current token is not equals to tk.
//	Take next token.
//
//==========================================================================

void VLexer::Expect(EToken tk, ECompileError error)
{
	if (Token != tk)
	{
		ParseError(Location, error, "expected %s, found %s", TokenNames[tk],
			TokenNames[Token]);
	}
	NextToken();
}

//==========================================================================
//
//	TLocation::GetSource
//
//==========================================================================

const char* TLocation::GetSource() const
{
	if (!Loc)
		return "(external)";
	return SourceFiles[Loc >> 16];
}