About to overhaul WingTypes

Plugins/UEWingman/Source/UEWingman/Private/WingTokenizer.cpp

@@ -44,11 +44,11 @@ WingCharacterClasses::WingCharacterClasses()
 
 WingCharacterClasses WingCharacterClasses::TheSet;
 
-void WingTokenizer::Add(TCHAR Type, FName InternalID)
+void WingTokenizer::Add(TCHAR Type, FName Name)
 {
 	Token T;
 	T.Type = Type;
-	T.InternalID = InternalID;
+	T.Name = Name;
 	Tokens.Add(T);
 }
 
@@ -211,6 +211,11 @@ WingTokenizer::WingTokenizer(const FString& In)
 		continue;
 	}
 	if (!Error.IsEmpty()) Tokens.Empty();
+
+	// Two sentinels means we can safely do lookahead 2 without risk.
+	Add(0, FName());
+	Add(0, FName());
+	Next = Tokens.GetData();
 }
 
 void WingTokenizer::PrintEverything(FStringBuilderBase &Out) const
@@ -224,7 +229,7 @@ void WingTokenizer::PrintEverything(FStringBuilderBase &Out) const
 		Out.Appendf(TEXT("Token '%c': "), T.Type);
 		if (T.Type == Identifier)
 		{
-			for (TCHAR Ch : T.InternalID.ToString())
+			for (TCHAR Ch : T.Name.ToString())
 			{
 				if (Ch >= 0x20 && Ch <= 0x7E)
 				{

Plugins/UEWingman/Source/UEWingman/Public/WingTokenizer.h

@@ -97,37 +97,47 @@ private:
 
 struct WingTokenizer
 {
-	using Cat = WingCharacterClasses::Cat;
+	//////////////////////////////////////////////////////////////////////////
+	//
+	// Tokenizer Accessors.
+	//
+	//////////////////////////////////////////////////////////////////////////
+
 	const TCHAR Identifier = 'i';
 	const TCHAR RestOfLine = 'r';
 
-	// A token has a token type which can be Identifier,
-	// RestOfLine, or a single-character punctuation mark.
-	// The InternalID field contains the result of converting
-	// the token from an external ID to an internal ID.
-	// Rest is only populated if it's a rest-of-line token.
-	struct Token
-	{
-		TCHAR Type;
-		FName InternalID;
-		FStringView Rest;
-	};
+	// Get the next token.
+	TCHAR NextType() const { return Next[0].Type; }
+	FName NextName() const { return Next[0].Name; }
+	FStringView NextRest() const { return Next[0].Rest; }
 
-	// The string that we tokenized.
-	FString Input;
+	// Check the next token.
+	bool TokenIs(TCHAR Type) const { return Next[0].Type == Type; }
 
-	// If the tokenization failed, an error message.
-	FString Error;
+	// Check the next token.
+	bool TokenIs(FName Name) const { return Next[0].Name == Name; }
 
-	// The result, an array of tokens.
-	TArray<Token> Tokens;
+	// Check the next two tokens.
+	bool TokenIs(FName Name, TCHAR Type) const { return Next[0].Name == Name && Next[1].Type == Type; }
+
+	// Advance the cursor.  Don't move past the two sentinels.
+	void Advance() { int I = Next-Tokens.GetData(); if (I + 2 < Tokens.Num()) Next++; }
 	
 	// Tokenize a line of input.  The tokens are stored in
-	// the token array.  If there's an error, the error is
-	// stored in the error field, and the token array is
-	// cleared.  If the tokens contain identifiers, 
+	// the token array, and the cursor is positioned on the first
+	// token.  If there's an error, the error is stored in the
+	// error field and the cursor points to the empty token. 
 	WingTokenizer(const FString& Input);
 
+	// Print all tokens into a string builder for debugging.
+	void PrintEverything(FStringBuilderBase &Out) const;
+
+	//////////////////////////////////////////////////////////////////////////
+	//
+	// Static functions for internalizing an externalizing IDs.
+	//
+	//////////////////////////////////////////////////////////////////////////
+
 	// Convert an internal ID into an external ID.
 	// Spaces are converted to periods.  Any other
 	// non-identifier character is HTML escaped.
@@ -153,10 +163,40 @@ struct WingTokenizer
 	// that's OK.
 	static FString SimplifyID(const FString &ID);
 
-	// Print all tokens into a string builder for debugging.
-	void PrintEverything(FStringBuilderBase &Out) const;
-
 private:
+	//////////////////////////////////////////////////////////////////////////
+	//
+	// The actual tokenizer, state.
+	//
+	//////////////////////////////////////////////////////////////////////////
+
+	struct Token
+	{
+		FName Name;        // Only if it's an identifier token.
+		FStringView Rest;  // Only if it's a rest-of-line token.
+		TCHAR Type = 0;    // A punctuation mark, or Identifier, or RestOfLine
+	};
+
+	// The string that we tokenized.
+	FString Input;
+
+	// If the tokenization failed, an error message.
+	FString Error;
+
+	// The result, an array of tokens.  Two empty sentinels at the end.
+	TArray<Token, TInlineAllocator<50>> Tokens;
+
+	// The cursor which advances along the tokens.  Never moves past sentinels.
+	Token *Next;
+
+	//////////////////////////////////////////////////////////////////////////
+	//
+	// Internal Implementation Functions.
+	//
+	//////////////////////////////////////////////////////////////////////////
+
+	using Cat = WingCharacterClasses::Cat;
+
 	// Add a token to the token array.
 	void Add(TCHAR Type, FName InternalID);
 	void Add(TCHAR Type, FStringView Rest);

Plugins/UEWingman/Source/UEWingman/Public/WingVariables.h

@@ -41,7 +41,6 @@ public:
     // Returns the set of flags that are supported by this variable category.
     static const TSet<FName> &GetRelevantFlagSet(Cat C);
     
-
     // Check if the variable list is empty.
     bool IsEmpty() const { return Variables.IsEmpty(); }
 

Plugins/UEWingman/Source/UEWingman/Public/WingWidgets.h

@@ -7,7 +7,6 @@ class UWidgetTree;
 struct FAssetData;
 
 
-// Utility functions for widget blueprint manipulation.
 class WingWidgets
 {
 public: