Modify LuaToken so that it emits in base36. The emit and parse routines are 64-bit clean, ie, they can parse and emit any 64-bit number in base36.

luprex/cpp/core/luastack.cpp

@@ -44,15 +44,20 @@ LuaConstantReg *LuaConstantReg::All;
 
 
 eng::string LuaToken::str() const {
-    uint64_t token = (uint64_t)value;
-    char buffer[9];
-    for (int i = 0; i < 8; i++) {
-        unsigned char c = token;
-        buffer[7-i] = c;
-        token >>= 8;
+    uint64_t n = (uint64_t)value;
+    char buffer[20];
+    int pos = 19;
+    buffer[pos] = 0;
+    while (n > 0) {
+        uint64_t digit = (n % 36);
+        n /= 36;
+        if (digit < 10) {
+            buffer[--pos] = '0' + digit;
+        } else {
+            buffer[--pos] = 'a' + (digit - 10);
+        }
     }
-    buffer[8] = 0;
-    return eng::string(buffer);
+    return eng::string(buffer + pos, 19 - pos);
 }
 
 static int panicf(lua_State *L) {

luprex/cpp/core/luastack.hpp

@@ -407,6 +407,41 @@ enum LuaTableType {
 ////////////////////////////////////////////////////////////////////
 
 struct LuaToken {
+public:
+    // Convert a base36 number into a 64-bit unsigned integer.  If the
+    // base36 number is not valid, or if it exceeds 64 bits, then return
+    // nullopt.  Leading zeros are not allowed.  Empty string gets parsed
+    // as zero.
+    //
+    static constexpr std::optional<uint64_t> parse_base36(const char *str) {
+        uint64_t result = 0;
+        uint64_t maxint = uint64_t(-1);
+        // Leading zeros are not allowed.
+        // Note: Token(0) is emitted as the empty string, not as '0'.
+        if ((*str == '0')) return std::nullopt;
+        while (*str) {
+            char c = *str++;
+            uint64_t digit = 0;
+            if ((c >= '0') && (c <= '9')) {
+                digit = uint64_t(c - '0');
+            } else if ((c >= 'a') && (c <= 'z')) {
+                digit = uint64_t(c - 'a' + 10);
+            } else if ((c >= 'A') && (c <= 'Z')) {
+                digit = uint64_t(c - 'A' + 10);
+            } else {
+                // Invalid digits return nullopt.
+                return std::nullopt;
+            }
+            // Multiply existing number by 36, then add the digit.
+            // We have two checks to prevent integer overflow.
+            if (result > (maxint / 36)) return std::nullopt;
+            result *= 36;
+            if (digit > (maxint - result)) return std::nullopt;
+            result += digit;
+        }
+        return result;
+    }
+
 public:
     uint64_t value;
 
@@ -417,7 +452,19 @@ public:
 
     // Construct a token from a short string.
     //
-    constexpr LuaToken(const char *str) : value(literal_to_token(str)) {}
+    // This is meant to be used for constants in the code.
+    // It cannot be used for runtime creation of tokens.  It is
+    // limited to 10-character tokens.
+    // 
+    // Note: using 'throw' in a consteval gives largely the
+    // effect of a static_assert.
+    //
+    consteval LuaToken(const char *str) : value(0) {
+        auto opt = parse_base36(str);
+        if (!opt.has_value()) throw "not a valid token";
+        if (opt.value_or(0) > 3656158440062975) throw "token may have at most 10 characters";
+        value = opt.value_or(0);
+    }
 
     // Construct a token from an int64.
     //
@@ -431,6 +478,9 @@ public:
     //
     LuaToken() : value(0) {}
 
+    // Assignment operator.
+    void operator =(const LuaToken &other) { value = other.value; }
+
     // Empty: return true if the token is all zero bytes.
     //
     bool empty() const { return value == 0; }
@@ -445,18 +495,13 @@ public:
 
     // Convert the token to a string.
     //
+    // The conversion to string consists of expressing the value
+    // in base 36.  The value 0 is expressed as the empty string.
+    //
     eng::string str() const;
 
-private:
-    static constexpr uint64_t literal_to_token(const char *str) {
-        uint64_t result = 0;
-        for (int i = 0; i < 8; i++) {
-            unsigned char c = *str;
-            result = (result << 8) + c;
-            if (*str) str++;
-        }
-        return result;
-    }
+public:
+
 };
 
 ////////////////////////////////////////////////////////////////////