#include "WingTokenizer.h"


void WingCharacterClasses::Assign(Cat Category, FStringView String)
{
	int32 OldSize = CharCategory.Num();
	int32 NewSize = 0;
	for (TCHAR Ch : String) if (Ch >= NewSize) NewSize = Ch + 1;
	if (NewSize > OldSize)
	{
		CharCategory.SetNum(NewSize);
		for (int i = OldSize; i < NewSize; i++) CharCategory[i] = Cat::Other;
	}
	for (TCHAR Ch : String) CharCategory[Ch] = Category;
}

WingCharacterClasses::WingCharacterClasses()
{
	// This is the set of printable, visible, non-whitespace characters that
	// appear in most ubuntu default fonts.  I initially map all of these as
	// 'Identifier' characters, but later I swap some of them over to punctuation.
	Assign(Cat::Identifier, TEXT(
		"!#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[]^_`abcdefgh"
		"ijklmnopqrstuvwxyz{|}~¡¢£¤¥¦§¨©ª«¬®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐ"
		"ÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿĀāĂăĄąĆćĈĉĊċČčĎďĐđĒēĔĕĖ"
		"ėĘęĚěĜĝĞğĠġĢģĤĥĦħĨĩĪīĬĭĮįİıĲĳĴĵĶķĸĹĺĻļĽľĿŀŁłŃńŅņŇňŉŊŋŌōŎŏŐőŒœŔŕŖŗŘřŚśŜ"
		"ŝŞşŠšŢţŤťŦŧŨũŪūŬŭŮůŰűŲųŴŵŶŷŸŹźŻżŽžſƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏƐƑƒƓƔƕƖƗƘƙƚƛƜƝƞƟƠơƢ"
		"ƣƤƥƦƧƨƩƪƫƬƭƮƯưƱƲƳƴƵƶƷƸƹƺƻƾƿǀǁǂǃǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟǠǡǢǣǦǧǨǩǪǫǬǭǮǯǰǴǵǸǹǼ"
		"ǽǾǿȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗȘșȚțȜȝȞȟȤȥȦȧȨȩȪȫȬȭȮȯȰȱȲȳȷʒʼˆˇˉ˘˙˚˛˜˝΄΅ΆΈΉΊ"
		"ΌΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίΰαβγδεζηθικλμνξοπρςστυφχψωϊϋόύώЀЁЂЃЄ"
		"ЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъ"
		"ыьэюяѐёђѓєѕіїјљњћќѝўџѲҐґҒғҔҕҖҗҘҙҚқҢңҤҥҪҫҬҭҮүҰұҲҳҺһӀӁӂӃӄӇӈӋӌӏӐӑӒӓӔӕӖӗӘә"
		"ӚӛӜӝӞӟӠӡӢӣӤӥӦӧӨөӪӫӬӭӮӯӰӱӲӳӴӵӶӷӸӹẀẁẂẃẄẅỲỳ–—―‘’‚“”„†‡•…‰‹›⁰⁴⁵⁶⁷⁸⁹₀₁₂₃₄₅₆"
		"₇₈₉€₹№™Ω⅓⅔⅕⅖⅗⅘⅙⅚⅛⅜⅝⅞∂∆∏∑−∕∙√∞∫≈≠≤≥◊ﬁﬂ\"\\"
	));

	// All the punctuation marks that we do anything interesting with.
	Assign(Cat::Punctuation, PunctuationString);

	// Control characters.
	for (int i = 0; i < 32; i++) CharCategory[i] = Cat::Control;
	CharCategory[0x7F] = Cat::Control;
}

WingCharacterClasses WingCharacterClasses::TheSet;

void WingTokenizer::Add(TCHAR Type, FName Name, FStringView Before, FStringView After)
{
	Token T;
	T.Type = Type;
	T.Name = Name;
	T.Source = FStringView(Before.GetData(), After.GetData() - Before.GetData());
	Tokens.Add(T);
}

void WingTokenizer::Add(TCHAR Type, FStringView Rest, FStringView Before, FStringView After)
{
	Token T;
	T.Type = Type;
	T.Rest = Rest;
	T.Source = FStringView(Before.GetData(), After.GetData() - Before.GetData());
	Tokens.Add(T);
}

TCHAR WingTokenizer::FromHex(FStringView Digits, FString &Error)
{
	if (Digits.Len() < 1) { Error = "Empty hex escape sequence"; return 0; }
	int32 Value = 0;
	for (TCHAR Ch : Digits)
	{
		if (Ch >= '0' && Ch <= '9') Value = Value * 16 + (Ch - '0');
		else if (Ch >= 'a' && Ch <= 'f') Value = Value * 16 + (Ch - 'a' + 10);
		else if (Ch >= 'A' && Ch <= 'F') Value = Value * 16 + (Ch - 'A' + 10);
		else { Error = "Invalid hex digit in escape sequence"; return 0; }
		if (Value > 0xFFFF) { Error = "Escape sequence value out of range"; return 0; }
	}
	return (TCHAR)Value;
}

TCHAR WingTokenizer::FromDecimal(FStringView Digits, FString &Error)
{
	if (Digits.Len() < 1) { Error = "Empty decimal escape sequence"; return 0; }
	int32 Value = 0;
	for (TCHAR Ch : Digits)
	{
		if (Ch >= '0' && Ch <= '9') Value = Value * 10 + (Ch - '0');
		else { Error = "Invalid decimal digit in escape sequence"; return 0; }
		if (Value > 0xFFFF) { Error = "Escape sequence value out of range"; return 0; }
	}
	return (TCHAR)Value;
}

TCHAR WingTokenizer::TokenizeEscapeSequence(FStringView &Rest, FString &Error)
{
	if (!Error.IsEmpty()) return 0;
	// Search for the semicolon.
	int32 SemiPos;
	if (!Rest.FindChar(';', SemiPos))
	{
		Error = "Ampersand escape sequence doesn't end in semicolon";
		return 0;
	}
	if (SemiPos < 3)
	{
		Error = "Ampersand escape sequence too short";
		return 0;
	}
	TCHAR Result = 0;
	if (Rest[1] == '#')
	{
		if ((Rest[2] == 'x') || (Rest[2] == 'X'))
			Result = FromHex(Rest.Mid(3, SemiPos - 3), Error);
		else
			Result = FromDecimal(Rest.Mid(2, SemiPos - 2), Error);
	}
	else
	{
		FString Name(Rest.Mid(1, SemiPos - 1));
		Result = WingEntityList::GetChar(Name);
		if (Result == 0)
		{
			Error = FString::Printf(TEXT("Unknown HTML entity: &%s;"), *Name);
			return 0;
		}
	}
	Rest = Rest.RightChop(SemiPos + 1);
	return Result;
}

FStringView WingTokenizer::TokenizeAssetName(FStringView &Rest, FString &Error)
{
	if (!Error.IsEmpty()) return FStringView();
	int i = 0;
	while ((i < Rest.Len()) && (Rest[i] != ',')) i++;
	FStringView Result = Rest.SubStr(0, i);
	Rest = Rest.RightChop(i);
	return Result;
}

FName WingTokenizer::TokenizeIdentifier(FStringView &Rest, FString &Error)
{
	if (!Error.IsEmpty()) return FName();
	TCHAR Buffer[NAME_SIZE];
	int Len = 0;
	while (!Rest.IsEmpty() && Error.IsEmpty())
	{
		TCHAR Ch = Rest[0];
		if (Ch == ' ') break;
		if (Ch == '.')
		{
			if (Len < NAME_SIZE) Buffer[Len++] = ' ';
			Rest = Rest.RightChop(1);
			continue;
		}
		if (Ch == '&')
		{
			TCHAR Decoded = TokenizeEscapeSequence(Rest, Error);
			if (Len < NAME_SIZE) Buffer[Len++] = Decoded;
			continue;
		}
		Cat Category = WingCharacterClasses::GetCat(Ch);
		if ((Category == Cat::Identifier) || (Category == Cat::Other))
		{
			// We accept other characters in case the LLM sends unicode
			// that isn't on the whitelist.  This is intentional.
			if (Len < NAME_SIZE) Buffer[Len++] = Ch;
			Rest = Rest.RightChop(1);
		}
		else break;
	}
	if (!Error.IsEmpty()) return FName();
	// The buffer has just enough room to hold the longest FName,
	// plus the required null terminator.  If we filled the whole
	// buffer, leaving no room for the null, it means the name
	// is too long.
	if (Len == NAME_SIZE)
	{
		Error = "FName too long";
		return FName();
	}
	Buffer[Len] = 0;
	return FName(Len, Buffer);
	// Note about code above: we deliberately do not check for empty
	// names here, because we don't have the context to generate a good
	// error message.  So instead, we leave it to the caller.
}

WingTokenizer::WingTokenizer(const FString& In)
{
	Input = In;
	FStringView Rest(Input);
	while (!Rest.IsEmpty() && Error.IsEmpty())
	{
		TCHAR Ch = Rest[0];
		if ((Ch == ' ') || (Ch == '\t'))
		{
			Rest = Rest.RightChop(1);
			continue;
		}
		FStringView Before = Rest;
		if (Ch == '=')
		{
			FStringView Body = Rest.RightChop(1);
			Rest = Rest.RightChop(Rest.Len());
			Add(RestOfLine, Body, Before, Rest);
			break;
		}
		if (Ch == '/')
		{
			FStringView Asset = TokenizeAssetName(Rest, Error);
			Add(AssetName, Asset, Before, Rest);
			continue;
		}
		if ((Ch == '.') || (Ch == '&'))
		{
			FName Id = TokenizeIdentifier(Rest, Error);
			Add(Identifier, Id, Before, Rest);
			continue;
		}
		Cat Category = WingCharacterClasses::GetCat(Ch);
		if (Category == Cat::Punctuation)
		{
			Rest = Rest.RightChop(1);
			Add(Ch, FString(), Before, Rest);
			continue;
		}
		if (Category == Cat::Control)
		{
			Error = "Control characters in input, not allowed";
			break;
		}
		FName Id = TokenizeIdentifier(Rest, Error);
		Add(Identifier, Id, Before, Rest);
		continue;
	}
	if (!Error.IsEmpty()) Tokens.Empty();

	// Two sentinels means we can safely do lookahead 2 without risk.
	Rest = Rest.LeftChop(Rest.Len());
	Add(0, FName(), Rest, Rest);
	Add(0, FName(), Rest, Rest);
	Next = Tokens.GetData();
}

void WingTokenizer::SaveCursor(FName Name)
{
	int Cursor = Next - Tokens.GetData();
	for (auto &Pair : SavedCursor) if (Pair.Key == Name) { Pair.Value = Cursor; return; }
	SavedCursor.Emplace(Name, Cursor);
}

FString WingTokenizer::GetRange(FName SavePos, int Extra) const
{
	int Lo = 0;
	for (auto &Pair : SavedCursor) if (Pair.Key == SavePos) Lo = Pair.Value;
	int Hi = (Next - Tokens.GetData()) + Extra;
	Hi = FMath::Clamp(Hi, Lo, Tokens.Num());
	if (Lo >= Hi) return FString();
	const TCHAR* Start = Tokens[Lo].Source.GetData();
	const TCHAR* End = Tokens[Hi - 1].Source.GetData() + Tokens[Hi - 1].Source.Len();
	return FString(End - Start, Start);
}

void WingTokenizer::PrintEverything(FStringBuilderBase &Out) const
{
	if (!Error.IsEmpty())
	{
		Out.Appendf(TEXT("Error: %s\n"), *Error);
	}
	for (const Token& T : Tokens)
	{
		Out.Appendf(TEXT("Token '%c': "), T.Type);
		if (T.Type == Identifier)
		{
			for (TCHAR Ch : T.Name.ToString())
			{
				if (Ch >= 0x20 && Ch <= 0x7E)
				{
					Out.AppendChar(Ch);
					Out.AppendChar(' ');
				}
				else
				{
					Out.Appendf(TEXT("%04X "), (int32)Ch);
				}
			}
		}
		if ((T.Type == RestOfLine) || (T.Type == AssetName))
		{
			Out.Appendf(TEXT("[%s]"), *FString(T.Rest));
		}
		Out.AppendChar('\n');
	}
}

FString WingTokenizer::ExternalizeID(FName InternalID)
{
	TStringBuilder<512> Result;
	TCHAR Buffer[FName::StringBufferSize];
	int32 Len = InternalID.ToString(Buffer);
	for (TCHAR Ch : FStringView(Buffer, Len))
	{
		if (Ch == ' ') Result.AppendChar('.');
		else if (WingCharacterClasses::GetCat(Ch) == Cat::Identifier) Result.AppendChar(Ch);
		else
		{
			Result.AppendChar('&');
			FStringView Name = WingEntityList::GetName(Ch);
			if (Name.IsEmpty())
			{
				Result.AppendChar('#');
				Result.Appendf(TEXT("%d"), (int32)Ch);
			}
			else
			{
				Result.Append(Name);
			}
			Result.AppendChar(';');
		}
	}
	return Result.ToString();
}

bool WingTokenizer::WouldExternalizeReadably(FName InternalID)
{
	if (InternalID.IsNone()) return false;
	TCHAR Buffer[FName::StringBufferSize];
	int32 Len = InternalID.ToString(Buffer);
	for (TCHAR Ch : FStringView(Buffer, Len))
	{
		if (Ch == ' ') continue;
		if (WingCharacterClasses::GetCat(Ch) != Cat::Identifier) return false;
	}
	return true;
}

FName WingTokenizer::TryInternalizeID(const FString &ExternalID, FString &Error)
{
	FStringView Input(ExternalID);
	Error.Empty();
	FName InternalID = TokenizeIdentifier(Input, Error);
	// If there's already an error, annotate with context
	if (!Error.IsEmpty())
	{
		Error = FString::Printf(TEXT("ERROR parsing id %s: %s"), *ExternalID, *Error);
		return FName();
	}
	// If the identifier tokenizer stops before consuming the whole
	// input, then we need to generate an error message.  We do our best
	// to generate the most informative error possible.
	if (!Input.IsEmpty())
	{
		Cat Category = WingCharacterClasses::GetCat(Input[0]);
		if (Input[0] == ' ')
		{
			Error = FString::Printf(TEXT("ERROR parsing id %s: in ids, spaces must be escaped"), *ExternalID);
		}
		else if (Category == Cat::Punctuation)
		{
			Error = FString::Printf(TEXT("ERROR parsing id %s: in ids, these marks must be escaped: %s"),
				*ExternalID, WingCharacterClasses::PunctuationString);
		}
		else if (Category == Cat::Control)
		{
			Error = FString::Printf(TEXT("ERROR parsing id %s: in ids, control characters must be escaped"), *ExternalID);
		}
		else Error = FString::Printf(TEXT("ERROR parsing id %s: unparseable character in id"), *ExternalID);
		return FName();
	}
	// One last error case: empty input
	if (InternalID.IsNone())
	{
		Error = TEXT("ERROR: Empty identifiers are not allowed");
		return FName();
	}
	return InternalID;
}


FString WingTokenizer::SimplifyID(const FString &ID)
{
	TStringBuilder<512> Result;
	for (TCHAR Ch : ID)
	{
		if (WingCharacterClasses::GetCat(Ch) == Cat::Identifier)
			Result.AppendChar(Ch);
	}
	return Result.ToString();
}
