Better support for serialization and sorting of the Token data type

Docs/Tokens-A-New-Lua-Type.md

@@ -69,3 +69,12 @@ Tokens can get passed to Unreal in a variety of ways.  For example, in animation
 ## Usage
 
 Tokens are mainly intended as sentinels and special reserved values. The JSON null example above is the motivating case, but tokens can represent any short reserved constant the engine needs.
+
+## Serialization and Difference Transmission
+
+I believe that we properly serialize and difference transmit tokens.
+
+* the serialize_lua function handles tokens explicitly
+* the difference transmitter has code for tokens
+* eris always saved lightuserdata as 64-bit numbers
+

luprex/cpp/core/luastack.cpp

@@ -48,19 +48,17 @@ LuaConstantReg *LuaConstantReg::All;
 
 eng::string LuaToken::str() const {
     uint64_t n = (uint64_t)value;
-    char buffer[20];
+    char buffer[13] = {};
-    int pos = 19;
+    for (int i = 11; i >= 0; i--) {
-    buffer[pos] = 0;
+        int d = n % 37;
-    while (n > 0) {
+        n /= 37;
-        uint64_t digit = (n % 36);
+        if (d >= 1 && d <= 10) {
-        n /= 36;
+            buffer[i] = '0' + (d - 1);
-        if (digit < 10) {
+        } else if (d >= 11 && d <= 36) {
-            buffer[--pos] = '0' + digit;
+            buffer[i] = 'a' + (d - 11);
-        } else {
-            buffer[--pos] = 'a' + (digit - 10);
         }
     }
-    return eng::string(buffer + pos, 19 - pos);
+    return eng::string(buffer);
 }
 
 static int panicf(lua_State *L) {
@@ -612,7 +610,7 @@ bool LuaCoreStack::tangetclass(LuaSlot classobj, LuaSlot tab) {
 
 bool LuaCoreStack::issortablekey(LuaSlot s) const {
     int type = lua_type(L_, s);
-    return (type == LUA_TBOOLEAN) || (type == LUA_TNUMBER) || (type == LUA_TSTRING);
+    return (type == LUA_TBOOLEAN) || (type == LUA_TNUMBER) || (type == LUA_TSTRING) || (type == LUA_TLIGHTUSERDATA);
 }
 
 void LuaCoreStack::movesortablekey(LuaSlot key, LuaCoreStack &otherstack, LuaSlot otherslot) {
@@ -633,6 +631,10 @@ void LuaCoreStack::movesortablekey(LuaSlot key, LuaCoreStack &otherstack, LuaSlo
         lua_replace(otherstack.L_, otherslot);
         break;
     }
+    case LUA_TLIGHTUSERDATA:
+        lua_pushlightuserdata(otherstack.L_, lua_touserdata(L_, key));
+        lua_replace(otherstack.L_, otherslot);
+        break;
     default:
         assert(false && "movesortablekey: not a sortable key");
     }
@@ -960,3 +962,35 @@ LuaDefine(keywords_finalcheckthrow, "table", kwdoc) {
     return LS.result();
 }
 
+LuaDefine(unittests_token, "", "Unit tests for LuaToken encoding") {
+    // Test round-trip encoding for various strings.
+    LuaAssertStrEq(L, LuaToken("a").str(), "a");
+    LuaAssertStrEq(L, LuaToken("z").str(), "z");
+    LuaAssertStrEq(L, LuaToken("0").str(), "0");
+    LuaAssertStrEq(L, LuaToken("9").str(), "9");
+    LuaAssertStrEq(L, LuaToken("null").str(), "null");
+    LuaAssertStrEq(L, LuaToken("hello").str(), "hello");
+    LuaAssertStrEq(L, LuaToken("zzzzzzzzzzzz").str(), "zzzzzzzzzzzz");
+    LuaAssertStrEq(L, LuaToken("a0").str(), "a0");
+    LuaAssertStrEq(L, LuaToken("0a").str(), "0a");
+    LuaAssertStrEq(L, LuaToken("000000000000").str(), "000000000000");
+
+    // Test that empty/invalid strings produce the empty token.
+    LuaAssert(L, LuaToken(std::string_view("")).empty());
+    LuaAssert(L, LuaToken(std::string_view("hello world")).empty());
+    LuaAssert(L, LuaToken(std::string_view("aaaaaaaaaaaaa")).empty()); // 13 chars
+
+    // Test that numeric ordering matches lexicographic ordering.
+    LuaAssert(L, LuaToken("a").value > LuaToken("0").value);
+    LuaAssert(L, LuaToken("b").value > LuaToken("a").value);
+    LuaAssert(L, LuaToken("aa").value > LuaToken("a").value);
+    LuaAssert(L, LuaToken("ab").value > LuaToken("aa").value);
+    LuaAssert(L, LuaToken("b").value > LuaToken("az").value);
+    LuaAssert(L, LuaToken("ba").value > LuaToken("az").value);
+    LuaAssert(L, LuaToken("hello").value > LuaToken("hell").value);
+    LuaAssert(L, LuaToken("a0").value > LuaToken("a").value);
+    LuaAssert(L, LuaToken("a").value != LuaToken("a0").value);
+
+    return 0;
+}
+

luprex/cpp/core/luastack.hpp

@@ -261,9 +261,9 @@
 // we have a json null.
 //
 // So that finally brings me to what a "token" is.  A token is a lightuserdata
-// containing a short string encoded as a base36 number.  Tokens may only
+// containing a short string encoded as a fixed-width base37 number.  Tokens
-// contain the characters a-z and 0-9, and can be up to 12 characters long
+// may only contain the characters a-z and 0-9, and can be up to 12 characters
-// (since 36^12 fits in 64 bits).  In effect, it's a short string, but it's
+// long (since 37^12 fits in 64 bits).  In effect, it's a short string, but it's
 // a string that's distinguishable from a normal lua string.  It doesn't have
 // the same type as a lua string (it shows up as a lightuserdata).
 // The purpose of tokens is to represent special unique values, like json null.
@@ -271,7 +271,7 @@
 // To make working with tokens easy, I've created a C++ struct 'LuaToken'.
 // It stores an int64.  You can construct a LuaToken in two different ways:
 //
-//   LuaToken(0x10FAA9)
+//   LuaToken(0x3D5E30BCAF2EF663)
 //   LuaToken("null")
 //
 // Those are equivalent.  The second form is just as fast as the first,
@@ -408,34 +408,36 @@ enum LuaTableType {
 
 struct LuaToken {
 private:
-    // Convert a base36 number into a token.  If the base36 number is
+    // Encode a token string as a fixed-width base37 number.
-    // not valid, or if it exceeds 64 bits, then return zero.
+    // Each character is mapped to a digit 1-36 (0 means "no character"),
+    // and the result is: CH0*37^11 + CH1*37^10 + ... + CH11*37^0.
+    // This fixed-width encoding ensures that numeric ordering matches
+    // lexicographic ordering of the original strings.
+    // Returns zero if the string is empty, too long, or contains
+    // invalid characters.
     //
     static constexpr uint64_t parse(std::string_view str) {
+        if (str.size() > 12) return 0;
+        if (str.empty()) return 0;
+
         uint64_t result = 0;
-        uint64_t maxint = uint64_t(-1);
-
-        // Leading zeros are not allowed.
-        if ((!str.empty()) && (str[0]=='0')) return 0;
-
         for (int i = 0; i < int(str.size()); i++) {
             char c = str[i];
             uint64_t digit = 0;
             if ((c >= '0') && (c <= '9')) {
-                digit = uint64_t(c - '0');
+                digit = uint64_t(c - '0') + 1;
             } else if ((c >= 'a') && (c <= 'z')) {
-                digit = uint64_t(c - 'a' + 10);
+                digit = uint64_t(c - 'a') + 11;
             } else if ((c >= 'A') && (c <= 'Z')) {
-                digit = uint64_t(c - 'A' + 10);
+                digit = uint64_t(c - 'A') + 11;
             } else {
-                return maxint;
+                return 0;
             }
-            // Multiply existing number by 36, then add the digit.
+            result = result * 37 + digit;
-            // We have two checks to prevent integer overflow.
+        }
-            if (result > (maxint / 36)) return 0;
+        // Pad remaining positions with zeros (no character).
-            result *= 36;
+        for (int i = int(str.size()); i < 12; i++) {
-            if (digit > (maxint - result)) return 0;
+            result = result * 37;
-            result += digit;
         }
         return result;
     }
@@ -450,7 +452,7 @@ public:
 
     // Construct a token from a string.
     //
-    // If the string is not a valid base36 number, then this
+    // If the string is not a valid token, then this
     // initializes the token to the empty token (zero)
     //
     LuaToken(std::string_view s) : value(parse(s)) {}

luprex/cpp/core/table.cpp

@@ -794,15 +794,15 @@ LuaDefine(genlt, "obj1,obj2",
     "|* Numbers are compared in the obvious numeric manner."
     "|* Strings are compared alphabetically."
     "|* Booleans are compared with false being less than true."
+    "|* Tokens are compared alphabetically."
     "|* Tables are all considered equal to other tables."
     "|* Functions are all considered equal to other functions."
     "|* Coroutines are all considered equal to other coroutines."
     "|"
-    "|* Numbers are less than strings."
+    "|Values of different types are printed in this order:"
-    "|* Strings are less than booleans."
+    "|"
-    "|* Booleans are less than functions."
+    "|  Numbers, Strings, Booleans, Tokens,"
-    "|* Functions are less than coroutines."
+    "|  Functions, Threads, Tables, and NIL"
-    "|* Coroutines are less than tables."
     "|") {
     LuaArg o1,o2;
     LuaRet lt;

luprex/cpp/core/world-difftab.cpp

@@ -161,6 +161,7 @@ static void transmit_value_debug_string(StreamBuffer *sb, eng::ostringstream &os
     case LUA_TLIGHTUSERDATA: {
         LuaToken token(sb->read_uint64());
         oss << "[" << token.str() << "]";
+        return;
     }
     case LUA_TT_GENERAL: {
         oss << "table " << sb->read_int32();

luprex/ext/eris-master/src/lapi.c

@@ -329,13 +329,19 @@ LUA_API void lua_arith (lua_State *L, int op) {
   lua_unlock(L);
 }
 
+/* Numbers are first, to help out pretty-printers that want
+ * to print the keys of a table in order.  */
 static int sortorder(int t) {
     switch (t) {
-    case LUA_TNIL: return 0;
+    case LUA_TNUMBER: return 0;
-    case LUA_TNUMBER: return 1;
+    case LUA_TSTRING: return 1;
-    case LUA_TSTRING: return 2;
+    case LUA_TBOOLEAN: return 2;
-    case LUA_TBOOLEAN: return 3;
+    case LUA_TLIGHTUSERDATA: return 3;
-    case LUA_TTABLE: return 1000000;
+    case LUA_TFUNCTION: return 4;
+    case LUA_TTHREAD: return 5;
+    case LUA_TUSERDATA: return 6;
+    case LUA_TTABLE: return 7;
+    case LUA_TNIL:   return 1000000;
     default: return t+1000;
     }
 }
@@ -356,6 +362,8 @@ LUA_API int lua_genlt (lua_State *L, int index1, int index2) {
     i = luaV_lessthan(L, o1, o2);
   } else if (t1 == LUA_TBOOLEAN) {
     i = bvalue(o1) < bvalue(o2);
+  } else if (t1 == LUA_TLIGHTUSERDATA) {
+    i = pvalue(o1) < pvalue(o2);
   } else {
     return 0;
   }