diff --git a/Plugins/UEWingman/Source/UEWingman/Private/WingTokenizer.cpp b/Plugins/UEWingman/Source/UEWingman/Private/WingTokenizer.cpp index d0c19491..bea6773e 100644 --- a/Plugins/UEWingman/Source/UEWingman/Private/WingTokenizer.cpp +++ b/Plugins/UEWingman/Source/UEWingman/Private/WingTokenizer.cpp @@ -44,11 +44,11 @@ WingCharacterClasses::WingCharacterClasses() WingCharacterClasses WingCharacterClasses::TheSet; -void WingTokenizer::Add(TCHAR Type, FName InternalID) +void WingTokenizer::Add(TCHAR Type, FName Name) { Token T; T.Type = Type; - T.InternalID = InternalID; + T.Name = Name; Tokens.Add(T); } @@ -211,6 +211,11 @@ WingTokenizer::WingTokenizer(const FString& In) continue; } if (!Error.IsEmpty()) Tokens.Empty(); + + // Two sentinels means we can safely do lookahead 2 without risk. + Add(0, FName()); + Add(0, FName()); + Next = Tokens.GetData(); } void WingTokenizer::PrintEverything(FStringBuilderBase &Out) const @@ -224,7 +229,7 @@ void WingTokenizer::PrintEverything(FStringBuilderBase &Out) const Out.Appendf(TEXT("Token '%c': "), T.Type); if (T.Type == Identifier) { - for (TCHAR Ch : T.InternalID.ToString()) + for (TCHAR Ch : T.Name.ToString()) { if (Ch >= 0x20 && Ch <= 0x7E) { diff --git a/Plugins/UEWingman/Source/UEWingman/Public/WingTokenizer.h b/Plugins/UEWingman/Source/UEWingman/Public/WingTokenizer.h index bbc6814d..4fe0cd32 100644 --- a/Plugins/UEWingman/Source/UEWingman/Public/WingTokenizer.h +++ b/Plugins/UEWingman/Source/UEWingman/Public/WingTokenizer.h @@ -96,38 +96,48 @@ private: struct WingTokenizer -{ - using Cat = WingCharacterClasses::Cat; +{ + ////////////////////////////////////////////////////////////////////////// + // + // Tokenizer Accessors. + // + ////////////////////////////////////////////////////////////////////////// + const TCHAR Identifier = 'i'; const TCHAR RestOfLine = 'r'; - // A token has a token type which can be Identifier, - // RestOfLine, or a single-character punctuation mark. - // The InternalID field contains the result of converting - // the token from an external ID to an internal ID. - // Rest is only populated if it's a rest-of-line token. - struct Token - { - TCHAR Type; - FName InternalID; - FStringView Rest; - }; + // Get the next token. + TCHAR NextType() const { return Next[0].Type; } + FName NextName() const { return Next[0].Name; } + FStringView NextRest() const { return Next[0].Rest; } - // The string that we tokenized. - FString Input; + // Check the next token. + bool TokenIs(TCHAR Type) const { return Next[0].Type == Type; } - // If the tokenization failed, an error message. - FString Error; + // Check the next token. + bool TokenIs(FName Name) const { return Next[0].Name == Name; } - // The result, an array of tokens. - TArray Tokens; + // Check the next two tokens. + bool TokenIs(FName Name, TCHAR Type) const { return Next[0].Name == Name && Next[1].Type == Type; } + // Advance the cursor. Don't move past the two sentinels. + void Advance() { int I = Next-Tokens.GetData(); if (I + 2 < Tokens.Num()) Next++; } + // Tokenize a line of input. The tokens are stored in - // the token array. If there's an error, the error is - // stored in the error field, and the token array is - // cleared. If the tokens contain identifiers, + // the token array, and the cursor is positioned on the first + // token. If there's an error, the error is stored in the + // error field and the cursor points to the empty token. WingTokenizer(const FString& Input); + // Print all tokens into a string builder for debugging. + void PrintEverything(FStringBuilderBase &Out) const; + + ////////////////////////////////////////////////////////////////////////// + // + // Static functions for internalizing an externalizing IDs. + // + ////////////////////////////////////////////////////////////////////////// + // Convert an internal ID into an external ID. // Spaces are converted to periods. Any other // non-identifier character is HTML escaped. @@ -153,10 +163,40 @@ struct WingTokenizer // that's OK. static FString SimplifyID(const FString &ID); - // Print all tokens into a string builder for debugging. - void PrintEverything(FStringBuilderBase &Out) const; - private: + ////////////////////////////////////////////////////////////////////////// + // + // The actual tokenizer, state. + // + ////////////////////////////////////////////////////////////////////////// + + struct Token + { + FName Name; // Only if it's an identifier token. + FStringView Rest; // Only if it's a rest-of-line token. + TCHAR Type = 0; // A punctuation mark, or Identifier, or RestOfLine + }; + + // The string that we tokenized. + FString Input; + + // If the tokenization failed, an error message. + FString Error; + + // The result, an array of tokens. Two empty sentinels at the end. + TArray> Tokens; + + // The cursor which advances along the tokens. Never moves past sentinels. + Token *Next; + + ////////////////////////////////////////////////////////////////////////// + // + // Internal Implementation Functions. + // + ////////////////////////////////////////////////////////////////////////// + + using Cat = WingCharacterClasses::Cat; + // Add a token to the token array. void Add(TCHAR Type, FName InternalID); void Add(TCHAR Type, FStringView Rest); diff --git a/Plugins/UEWingman/Source/UEWingman/Public/WingVariables.h b/Plugins/UEWingman/Source/UEWingman/Public/WingVariables.h index 3b129aad..1526b2e3 100644 --- a/Plugins/UEWingman/Source/UEWingman/Public/WingVariables.h +++ b/Plugins/UEWingman/Source/UEWingman/Public/WingVariables.h @@ -41,7 +41,6 @@ public: // Returns the set of flags that are supported by this variable category. static const TSet &GetRelevantFlagSet(Cat C); - // Check if the variable list is empty. bool IsEmpty() const { return Variables.IsEmpty(); } diff --git a/Plugins/UEWingman/Source/UEWingman/Public/WingWidgets.h b/Plugins/UEWingman/Source/UEWingman/Public/WingWidgets.h index 45f00221..4d64f667 100644 --- a/Plugins/UEWingman/Source/UEWingman/Public/WingWidgets.h +++ b/Plugins/UEWingman/Source/UEWingman/Public/WingWidgets.h @@ -7,7 +7,6 @@ class UWidgetTree; struct FAssetData; -// Utility functions for widget blueprint manipulation. class WingWidgets { public: