Move some stuff out of LuaStack

Docs/Module-Dependencies-in-Luprex.md

@@ -14,6 +14,7 @@ the `.cpp` file (not the `.hpp`), it is marked **(cpp-only)**.
 - **debugcollector** → util
 - **streambuffer** → eng-malloc, luastack, util
 - **table** → luastack
+- **pprint** → luastack, table, util
 - **drivenengine** → enginewrapper, invocation, streambuffer, util
 - **json** → luastack, util
 - **http** → drivenengine, json(cpp-only), luastack, streambuffer
@@ -23,7 +24,6 @@ the `.cpp` file (not the `.hpp`), it is marked **(cpp-only)**.
 - **sched** → luastack, streambuffer
 - **idalloc** → debugcollector, luastack, streambuffer
 - **invocation** → enginewrapper, streambuffer
-- **pprint** → luastack, table, util
 - **source** → debugcollector, luastack, luasnap, streambuffer, table(cpp-only), traceback, util
 - **animqueue** → debugcollector, luastack, streambuffer, util
 - **printbuffer** → debugcollector, invocation, streambuffer, util

luprex/Makefile

@@ -82,7 +82,7 @@ BASE_ERIS := \
 BASE_CORE := \
 	invocation spookyv2 eng-malloc debugcollector drivenengine util luastack \
 	traceback planemap pprint luavector idalloc sched http \
-	json table luasnap animqueue streambuffer source world-core world-accessor \
+	json table luasnap animqueue streambuffer source keywords world-core world-accessor \
 	world-difftab world-diffxmit world-pairtab world-testing lpxserver lpxclient \
 	unit-testing printbuffer serializelua
 

luprex/cpp/core/http.hpp

@@ -18,6 +18,7 @@
 #include "wrap-vector.hpp"
 #include "wrap-map.hpp"
 #include "luastack.hpp"
+#include "keywords.hpp"
 #include "streambuffer.hpp"
 #include "drivenengine.hpp"
 #include <ostream>

luprex/cpp/core/keywords.cpp

@@ -0,0 +1,234 @@
+#include "wrap-string.hpp"
+
+#include "keywords.hpp"
+#include "util.hpp"
+
+static LuaToken token_error("error");
+static LuaToken token_found("found");
+
+LuaKeywordParser::LuaKeywordParser(const LuaCoreStack &LS0, LuaSlot slot)
+    : keytab(slot.index()), LS(LS0.state()) {
+    lua_State *L = LS0.state();
+    lua_pushnil(L); found.index_ = lua_gettop(L);
+    lua_pushnil(L); error.index_ = lua_gettop(L);
+    lua_pushnil(L); key.index_ = lua_gettop(L);
+    lua_pushnil(L); val.index_ = lua_gettop(L);
+
+    istable = LS.istable(keytab);
+    if (istable) {
+        LS.rawget(found, keytab, token_found);
+        if (!LS.istable(found)) {
+            LS.set(found, LuaNewTable);
+            LS.rawset(keytab, token_found, found);
+        }
+    }
+}
+
+bool LuaKeywordParser::optional(LuaSlot out, std::string_view kw) {
+    if (!istable) {
+        LS.set(out, LuaNil);
+        return false;
+    }
+
+    LS.rawget(out, keytab, kw);
+    if (!LS.isnil(out)) {
+        LS.rawset(found, kw, true);
+        return true;
+    } else {
+        return false;
+    }
+};
+
+bool LuaKeywordParser::required(LuaSlot out, std::string_view kw) {
+    if (!istable) {
+        LS.set(out, LuaNil);
+        return false;
+    }
+
+    LS.rawget(out, keytab, kw);
+    if (!LS.isnil(out)) {
+        LS.rawset(found, kw, true);
+        return true;
+    } else {
+        LS.rawget(error, keytab, token_error);
+        if (!LS.isstring(error)) {
+            LS.rawset(keytab, token_error, util::ss("required keyword argument not present: ", kw));
+        }
+        return false;
+    }
+};
+
+eng::string LuaKeywordParser::check() {
+    if (!istable) {
+        return "keyword arguments must be a table";
+    }
+
+    LS.rawget(error, keytab, token_error);
+    auto str = LS.trystring(error);
+    if (str.has_value()) return str.value();
+
+    return "";
+}
+
+eng::string LuaKeywordParser::final_check() {
+    if (!istable) {
+        return "keyword arguments must be a table";
+    }
+
+    LS.rawget(error, keytab, token_error);
+    auto str = LS.trystring(error);
+    if (str.has_value()) return str.value();
+
+    LS.set(key, LuaNil);
+    while (LS.next(keytab, key, val)) {
+        if (LS.istoken(key)) {
+            continue;
+        }
+        auto kw = LS.trystringview(key);
+        if (!kw.has_value()) {
+            return "keyword arguments include a non-string key";
+        }
+        LS.rawget(val, found, key);
+        if (!LS.rawequal(val, true)) {
+            return util::ss("unrecognized keyword argument: ", kw.value());
+        }
+    }
+    return "";
+}
+
+void LuaKeywordParser::check_throw() {
+    eng::string err = check();
+    if (!err.empty()) {
+        luaL_error(LS.state(), "%s", err.c_str());
+    }
+}
+
+void LuaKeywordParser::final_check_throw() {
+    eng::string err = final_check();
+    if (!err.empty()) {
+        luaL_error(LS.state(), "%s", err.c_str());
+    }
+}
+
+static const char *kwdoc =
+    "|Parse lua keyword arguments."
+    "|"
+    "|The keywords module is used to help parse keyword arguments"
+    "|for functions.  The error handling is strong, it throws useful"
+    "|error messages when the caller passes in incorrect arguments."
+    "|This is a typical example of how it is used:"
+    "|"
+    "|   function drawrect(args)"
+    "|      local x1,y1 = keywords.required(args, 'x1', 'y1')"
+    "|      local x2,y2 = keywords.required(args, 'x2', 'y2')"
+    "|      local color = keywords.optional(args, 'color')"
+    "|      keywords.finalcheckthrow(args)"
+    "|      ..."
+    "|   end"
+    "|"
+    "|The function above expects four required keyword arguments and one"
+    "|optional one.  The example code above handles all the following"
+    "|error cases:"
+    "|"
+    "|  * That 'args' actually is a table."
+    "|  * That the required keywords x1,y1,x2,y2 are all present."
+    "|  * That the table doesn't contain extraneous unknown keywords."
+    "|"
+    "|The functions keywords.required and keywords.optional don't throw"
+    "|lua errors.  Instead, they store status information in the keyword"
+    "|table itself.  Using the status information, the function"
+    "|keywords.finalcheckthrow will throw an error if anything went wrong"
+    "|along the way."
+    "|"
+    "|You can check errors when you're not finished parsing using the"
+    "|function keywords.checkthrow() instead of keywords.finalcheckthrow()."
+    "|Unlike finalcheckthrow, this function doesn't verify the absence of"
+    "|extraneous keywords, which can only be done at the end."
+    "|"
+    "|If you don't want to throw errors at all, you can use"
+    "|keywords.finalcheck() instead of keywords.finalcheckthrow()."
+    "|This returns the error message, rather than throwing it."
+    "|";
+
+
+LuaDefine(keywords_optional, "table, keyword, keyword, keyword...", kwdoc) {
+    LuaArg table;
+    LuaExtraArgs keywords;
+    LuaDefStack LS(L, table, keywords);
+    LuaKeywordParser KP(LS, table);
+
+    if (keywords.size() < 1) {
+        luaL_error(L, "expected at least one keyword");
+        return 0;
+    }
+    for (int i = 0; i < keywords.size(); i++) {
+        eng::string kw = LS.ckstring(keywords[i], "keyword");
+        KP.optional(keywords[i], kw);
+    }
+
+    // Return the results without using LS.result, because it doesn't
+    // support multiple return values.
+    lua_settop(L, keywords[keywords.size() - 1].index());
+    return keywords.size();
+}
+
+LuaDefine(keywords_required, "table, keyword, keyword, keyword...", kwdoc) {
+    LuaArg table;
+    LuaExtraArgs keywords;
+    LuaDefStack LS(L, table, keywords);
+    LuaKeywordParser KP(LS, table);
+
+    if (keywords.size() < 1) {
+        luaL_error(L, "expected at least one keyword");
+        return 0;
+    }
+    for (int i = 0; i < keywords.size(); i++) {
+        eng::string kw = LS.ckstring(keywords[i], "keyword");
+        KP.required(keywords[i], kw);
+    }
+
+    // Return the results without using LS.result, because it doesn't
+    // support multiple return values.
+    lua_settop(L, keywords[keywords.size() - 1].index());
+    return keywords.size();
+}
+
+LuaDefine(keywords_check, "table", kwdoc) {
+    LuaArg table;
+    LuaRet result;
+    LuaDefStack LS(L, table, result);
+    LuaKeywordParser KP(LS, table);
+    eng::string err = KP.check();
+    if (!err.empty()) {
+        LS.set(result, err);
+    }
+    return LS.result();
+}
+
+LuaDefine(keywords_finalcheck, "table", kwdoc) {
+    LuaArg table;
+    LuaRet result;
+    LuaDefStack LS(L, table, result);
+    LuaKeywordParser KP(LS, table);
+    eng::string err = KP.final_check();
+    if (!err.empty()) {
+        LS.set(result, err);
+    }
+    return LS.result();
+}
+
+LuaDefine(keywords_checkthrow, "table", kwdoc) {
+    LuaArg table;
+    LuaDefStack LS(L, table);
+    LuaKeywordParser KP(LS, table);
+    KP.check_throw();
+    return LS.result();
+}
+
+LuaDefine(keywords_finalcheckthrow, "table", kwdoc) {
+    LuaArg table;
+    LuaDefStack LS(L, table);
+    LuaKeywordParser KP(LS, table);
+    KP.final_check_throw();
+    return LS.result();
+}

luprex/cpp/core/keywords.hpp

@@ -0,0 +1,92 @@
+////////////////////////////////////////////////////////////////////
+//
+// LuaKeywordParser
+//
+// This is a helper class to help parse tables full of keywords.
+// It is meant to make it easier to write LuaDefine functions that
+// accept keyword arguments.  It helps with the following tasks:
+//
+// * It makes sure the keyword table actually is a table.
+//
+// * It makes sure that all required keywords are present.
+//
+// * It makes sure that you didn't put anything that isn't a
+//   known keyword into the keyword table.
+//
+// This module adds two fields to the table:
+//
+//  [ERROR] - stores an error message, initially nil.
+//  [FOUND] - the set of keywords successfully parsed.
+//
+// If at any time, this module detects an error, it doesn't throw.
+// Instead, it stores an error report in the table key [ERROR].
+// Later, you can check for an error string using the function
+// final_check or final_check_throw.  If an error is
+// detected when there is already an error report in the table,
+// the error report is not overwritten, so therefore, the error
+// reported is always the first error detected.
+//
+// When this module finds a keyword in the table, it adds the keyword
+// to the [FOUND] set of all keywords successfully parsed.
+//
+// If the keyword table that you pass in isn't a table at all,
+// then the keyword-fetching functions will always return false.
+// Later, when you call 'check', an appropriate error will be
+// generated.
+//
+// The lua module 'keywords' contains the same functions as this
+// C++ class.  You can write code where a C++ function does some
+// of the parsing, and the lua code does the rest of the parsing.
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef KEYWORDS_HPP
+#define KEYWORDS_HPP
+
+#include "luastack.hpp"
+
+class LuaKeywordParser {
+private:
+    LuaVar found, error, key, val;
+    LuaSpecial keytab;
+    LuaCoreStack LS;
+    bool istable;
+
+    void init(const lua_State *L, int slot);
+public:
+    LuaKeywordParser(const LuaCoreStack &LS, LuaSlot slot);
+
+    // Fetch the value of the keyword.  If the keyword is found, then the
+    // keyword is added to the [FOUND] set, the value is returned in slot,
+    // and returns true.  Otherwise, sets slot to nil and returns false.
+    bool optional(LuaSlot slot, std::string_view kw);
+
+    // Fetch the value of the keyword.  If the keyword is found, then the
+    // keyword is added to the [FOUND] set, the value is returned in slot,
+    // and returns true.  Otherwise, sets slot to nil, returns false, and
+    // stores an [ERROR] report in the keyword table.
+    bool required(LuaSlot slot, std::string_view kw);
+
+    // Check if there are any errors so far, by checking for an [ERROR]
+    // report in the keyword table.  If any error has been
+    // detected, returns an error message, otherwise, returns empty
+    // string.
+    eng::string check();
+
+    // Check if there are any errors so far, by checking for an [ERROR]
+    // report in the keyword table.  Also check that all keyword
+    // arguments present in the table are in the [FOUND] set.  If there are
+    // any errors, returns an error message, otherwise returns empty string.
+    eng::string final_check();
+
+    // If check() returns an error, throws the error using luaL_error.
+    void check_throw();
+
+    // If final_check() returns an error, throws the error using luaL_error.
+    void final_check_throw();
+
+    // Fetch the state pointer.
+    lua_State *state() const { return LS.state(); }
+};
+
+#endif // KEYWORDS_HPP

luprex/cpp/core/luasnap.cpp

@@ -8,6 +8,21 @@
 
 #include <cassert>
 
+class LuaByteReader {
+private:
+    const char *data_;
+    int64_t size_;
+public:
+    LuaByteReader(const char *d, int64_t s) : data_(d), size_(s) {}
+    static const char *lua_reader(lua_State *L, void *ud, size_t *size) {
+        LuaByteReader *reader = (LuaByteReader*)ud;
+        *size = reader->size_;
+        const char *data = reader->data_;
+        reader->data_ = 0;
+        reader->size_ = 0;
+        return data;
+    }
+};
 
 LuaSnap::LuaSnap() {
     state_ = LuaCoreStack::newstate(eng::l_alloc);
@@ -86,7 +101,7 @@ void LuaSnap::deserialize(StreamBuffer *sb) {
     // Lua stack should be empty.
     assert(lua_gettop(state_) == 0);
 
-    // Get the eris data and convert it to a LuaByteReader.
+    // Get the eris data and convert it to a byte reader.
     int64_t eris_len = sb->read_int64();
     const char *eris_bytes = sb->read_bytes(eris_len);
     LuaByteReader bytereader(eris_bytes, eris_len);
@@ -94,7 +109,7 @@ void LuaSnap::deserialize(StreamBuffer *sb) {
     // Call eris with the permanents table and passing the snapshot as a lua_Reader.
     lua_pushstring(state_, "unpersist");
     lua_rawget(state_, LUA_REGISTRYINDEX);
-    eris_undump(state_, bytereader.lua_reader, bytereader.lua_reader_userdata());
+    eris_undump(state_, bytereader.lua_reader, &bytereader);
 
     // Set up a stack frame manually.  Eris deserialization
     // should have left permstable and regcopy on the stack.

luprex/cpp/core/luastack.cpp

@@ -10,9 +10,6 @@
 LuaSpecial LuaRegistry(LUA_REGISTRYINDEX);
 LuaNilMarker LuaNil;
 LuaNewTableMarker LuaNewTable;
-static LuaToken token_error("error");
-static LuaToken token_found("found");
-
 
 LuaFunctionReg::LuaFunctionReg(const char *n, const char *a, const char *d, bool s, lua_CFunction f) {
     name_ = n;
@@ -46,18 +43,6 @@ LuaFunctionReg *LuaFunctionReg::All;
 LuaConstantReg *LuaConstantReg::All;
 
 
-eng::string LuaToken::str() const {
-    static const char encoding[] =
-        "\0_0123456789abcdefghijklmnopqrstuvwxyz";
-    uint64_t n = (uint64_t)value;
-    char buffer[13] = {};
-    for (int i = 11; i >= 0; i--) {
-        buffer[i] = encoding[n % 38];
-        n /= 38;
-    }
-    return eng::string(buffer);
-}
-
 static int panicf(lua_State *L) {
     const char *p = lua_tostring(L, -1);
     fprintf(stderr, "PANIC: unprotected error in call to Lua API (%s)\n", p);
@@ -680,242 +665,6 @@ int LuaDefStack::tailcall_internal(bool passup, int base, int nargs) {
     return lua_gettop(L_) - base;
 }
 
-LuaKeywordParser::LuaKeywordParser(const LuaCoreStack &LS0, LuaSlot slot)
-    : keytab(slot.index()), LS(LS0.state()) {
-    lua_State *L = LS0.state();
-    lua_pushnil(L); found.index_ = lua_gettop(L);
-    lua_pushnil(L); error.index_ = lua_gettop(L);
-    lua_pushnil(L); key.index_ = lua_gettop(L);
-    lua_pushnil(L); val.index_ = lua_gettop(L);
-    
-    istable = LS.istable(keytab);
-    if (istable) {
-        LS.rawget(found, keytab, token_found);
-        if (!LS.istable(found)) {
-            LS.set(found, LuaNewTable);
-            LS.rawset(keytab, token_found, found);
-        }
-    }
-}
-
-bool LuaKeywordParser::optional(LuaSlot out, std::string_view kw) {
-    if (!istable) {
-        LS.set(out, LuaNil);
-        return false;
-    }
-
-    LS.rawget(out, keytab, kw);
-    if (!LS.isnil(out)) {
-        LS.rawset(found, kw, true);
-        return true;
-    } else {
-        return false;
-    }
-};
-
-bool LuaKeywordParser::required(LuaSlot out, std::string_view kw) {
-    if (!istable) {
-        LS.set(out, LuaNil);
-        return false;
-    }
-
-    LS.rawget(out, keytab, kw);
-    if (!LS.isnil(out)) {
-        LS.rawset(found, kw, true);
-        return true;
-    } else {
-        LS.rawget(error, keytab, token_error);
-        if (!LS.isstring(error)) {
-            LS.rawset(keytab, token_error, util::ss("required keyword argument not present: ", kw));
-        }
-        return false;
-    }
-};
-
-eng::string LuaKeywordParser::check() {
-    if (!istable) {
-        return "keyword arguments must be a table";
-    }
-
-    LS.rawget(error, keytab, token_error);
-    auto str = LS.trystring(error);
-    if (str.has_value()) return str.value();
-
-    return "";
-}
-
-eng::string LuaKeywordParser::final_check() {
-    if (!istable) {
-        return "keyword arguments must be a table";
-    }
-
-    LS.rawget(error, keytab, token_error);
-    auto str = LS.trystring(error);
-    if (str.has_value()) return str.value();
-
-    LS.set(key, LuaNil);
-    while (LS.next(keytab, key, val)) {
-        if (LS.istoken(key)) {
-            continue;
-        }
-        auto kw = LS.trystringview(key);
-        if (!kw.has_value()) {
-            return "keyword arguments include a non-string key";
-        }
-        LS.rawget(val, found, key);
-        if (!LS.rawequal(val, true)) {
-            return util::ss("unrecognized keyword argument: ", kw.value());
-        }
-    }
-    return "";
-}
-
-void LuaKeywordParser::check_throw() {
-    eng::string err = check();
-    if (!err.empty()) {
-        luaL_error(LS.state(), "%s", err.c_str());
-    }
-}
-
-void LuaKeywordParser::final_check_throw() {
-    eng::string err = final_check();
-    if (!err.empty()) {
-        luaL_error(LS.state(), "%s", err.c_str());
-    }
-}
-
-const char *LuaByteReader::lua_reader(lua_State *L, void *ud, size_t *size) {
-    LuaByteReader *reader = (LuaByteReader*)ud;
-    *size = reader->size_;
-    const char *data = reader->data_;
-    reader->data_ = 0;
-    reader->size_ = 0;
-    return data;
-}
-
-static const char *kwdoc = 
-    "|Parse lua keyword arguments."
-    "|"
-    "|The keywords module is used to help parse keyword arguments"
-    "|for functions.  The error handling is strong, it throws useful"
-    "|error messages when the caller passes in incorrect arguments."
-    "|This is a typical example of how it is used:"
-    "|"
-    "|   function drawrect(args)"
-    "|      local x1,y1 = keywords.required(args, 'x1', 'y1')"
-    "|      local x2,y2 = keywords.required(args, 'x2', 'y2')"
-    "|      local color = keywords.optional(args, 'color')"
-    "|      keywords.finalcheckthrow(args)"
-    "|      ..."
-    "|   end"
-    "|"
-    "|The function above expects four required keyword arguments and one"
-    "|optional one.  The example code above handles all the following"
-    "|error cases:"
-    "|"
-    "|  * That 'args' actually is a table."
-    "|  * That the required keywords x1,y1,x2,y2 are all present."
-    "|  * That the table doesn't contain extraneous unknown keywords."
-    "|"
-    "|The functions keywords.required and keywords.optional don't throw"
-    "|lua errors.  Instead, they store status information in the keyword"
-    "|table itself.  Using the status information, the function"
-    "|keywords.finalcheckthrow will throw an error if anything went wrong"
-    "|along the way."
-    "|"
-    "|You can check errors when you're not finished parsing using the"
-    "|function keywords.checkthrow() instead of keywords.finalcheckthrow()."
-    "|Unlike finalcheckthrow, this function doesn't verify the absence of"
-    "|extraneous keywords, which can only be done at the end."
-    "|"
-    "|If you don't want to throw errors at all, you can use"
-    "|keywords.finalcheck() instead of keywords.finalcheckthrow()."
-    "|This returns the error message, rather than throwing it."
-    "|";
-
-
-LuaDefine(keywords_optional, "table, keyword, keyword, keyword...", kwdoc) {    
-    LuaArg table;
-    LuaExtraArgs keywords;
-    LuaDefStack LS(L, table, keywords);
-    LuaKeywordParser KP(LS, table);
-
-    if (keywords.size() < 1) {
-        luaL_error(L, "expected at least one keyword");
-        return 0;
-    }
-    for (int i = 0; i < keywords.size(); i++) {
-        eng::string kw = LS.ckstring(keywords[i], "keyword");
-        KP.optional(keywords[i], kw);
-    }
-
-    // Return the results without using LS.result, because it doesn't
-    // support multiple return values.
-    lua_settop(L, keywords[keywords.size() - 1].index());
-    return keywords.size();
-}
-
-LuaDefine(keywords_required, "table, keyword, keyword, keyword...", kwdoc) {    
-    LuaArg table;
-    LuaExtraArgs keywords;
-    LuaDefStack LS(L, table, keywords);
-    LuaKeywordParser KP(LS, table);
-
-    if (keywords.size() < 1) {
-        luaL_error(L, "expected at least one keyword");
-        return 0;
-    }
-    for (int i = 0; i < keywords.size(); i++) {
-        eng::string kw = LS.ckstring(keywords[i], "keyword");
-        KP.required(keywords[i], kw);
-    }
-
-    // Return the results without using LS.result, because it doesn't
-    // support multiple return values.
-    lua_settop(L, keywords[keywords.size() - 1].index());
-    return keywords.size();
-}
-
-LuaDefine(keywords_check, "table", kwdoc) {
-    LuaArg table;
-    LuaRet result;
-    LuaDefStack LS(L, table, result);
-    LuaKeywordParser KP(LS, table);
-    eng::string err = KP.check();
-    if (!err.empty()) {
-        LS.set(result, err);
-    }
-    return LS.result();
-}
-
-LuaDefine(keywords_finalcheck, "table", kwdoc) {
-    LuaArg table;
-    LuaRet result;
-    LuaDefStack LS(L, table, result);
-    LuaKeywordParser KP(LS, table);
-    eng::string err = KP.final_check();
-    if (!err.empty()) {
-        LS.set(result, err);
-    }
-    return LS.result();
-}
-
-LuaDefine(keywords_checkthrow, "table", kwdoc) {
-    LuaArg table;
-    LuaDefStack LS(L, table);
-    LuaKeywordParser KP(LS, table);
-    KP.check_throw();
-    return LS.result();
-}
-
-LuaDefine(keywords_finalcheckthrow, "table", kwdoc) {
-    LuaArg table;
-    LuaDefStack LS(L, table);
-    LuaKeywordParser KP(LS, table);
-    KP.final_check_throw();
-    return LS.result();
-}
-
 LuaDefine(unittests_token, "", "Unit tests for LuaToken encoding") {
     // Test round-trip encoding for various strings.
     LuaAssertStrEq(L, LuaToken("a").str(), "a");

luprex/cpp/core/luastack.hpp

@@ -45,49 +45,6 @@ enum LuaTableType {
 ////////////////////////////////////////////////////////////////////
 
 struct LuaToken {
-private:
-    // Encode a token string as a fixed-width base38 number.
-    // Each character is mapped to a digit 1-37 (0 means "no character"),
-    // and the result is: CH0*38^11 + CH1*38^10 + ... + CH11*38^0.
-    // This fixed-width encoding ensures that numeric ordering matches
-    // lexicographic ordering of the original strings.
-    //
-    // Digit mapping: _ → 1, 0-9 → 2-11, a-z → 12-37.
-    //
-    // WARNING: The Lua lexer in llex.c contains a duplicate of this
-    // encoding logic (in the '@' token literal case). If you change
-    // the encoding here, you must update llex.c to match.
-    // Returns zero if the string is empty, too long, or contains
-    // invalid characters.
-    //
-    static constexpr uint64_t parse(std::string_view str) {
-        if (str.size() > 12) return 0;
-        if (str.empty()) return 0;
-
-        uint64_t result = 0;
-        for (int i = 0; i < int(str.size()); i++) {
-            char c = str[i];
-            uint64_t digit = 0;
-            if (c == '_') {
-                digit = 1;
-            } else if ((c >= '0') && (c <= '9')) {
-                digit = uint64_t(c - '0') + 2;
-            } else if ((c >= 'a') && (c <= 'z')) {
-                digit = uint64_t(c - 'a') + 12;
-            } else if ((c >= 'A') && (c <= 'Z')) {
-                digit = uint64_t(c - 'A') + 12;
-            } else {
-                return 0;
-            }
-            result = result * 38 + digit;
-        }
-        // Pad remaining positions with zeros (no character).
-        for (int i = int(str.size()); i < 12; i++) {
-            result = result * 38;
-        }
-        return result;
-    }
-
 public:
     uint64_t value;
 
@@ -101,8 +58,8 @@ public:
     // If the string is not a valid token, then this
     // initializes the token to the empty token (zero)
     //
-    LuaToken(std::string_view s) : value(parse(s)) {}
+    LuaToken(std::string_view s) : value(util::encode_token(s)) {}
-    LuaToken(const eng::string &s) : value(parse(s)) {}
+    LuaToken(const eng::string &s) : value(util::encode_token(s)) {}
 
     // Construct a token from a compile-time constant string.
     //
@@ -111,7 +68,11 @@ public:
     // consteval (evaluated at compile time), the error is
     // generated during the compilation.
     //
-    consteval LuaToken(const char *s) : value(parse(s)) {
+    // WARNING: The Lua lexer in llex.c contains a duplicate of the
+    // encoding logic (in the '@' token literal case). If you change
+    // the encoding in util::encode_token, you must update llex.c to match.
+    //
+    consteval LuaToken(const char *s) : value(util::encode_token(s)) {
         if (empty()) throw "cannot parse token";
     }
 
@@ -144,10 +105,7 @@ public:
 
     // Convert the token to a string.
     //
-    // The conversion to string consists of expressing the value
+    eng::string str() const { return util::decode_token(value); }
-    // in base 36.
-    //
-    eng::string str() const;
 };
 
 ////////////////////////////////////////////////////////////////////
@@ -898,113 +856,9 @@ public:
     }
 };
 
-////////////////////////////////////////////////////////////////////
-//
-// LuaKeywordParser
-//
-// This is a helper class to help parse tables full of keywords.
-// It is meant to make it easier to write LuaDefine functions that
-// accept keyword arguments.  It helps with the following tasks:
-//
-// * It makes sure the keyword table actually is a table.
-//
-// * It makes sure that all required keywords are present.
-//
-// * It makes sure that you didn't put anything that isn't a
-//   known keyword into the keyword table.
-//
-// This module adds two fields to the table:
-//
-//  [ERROR] - stores an error message, initially nil.
-//  [FOUND] - the set of keywords successfully parsed.
-//
-// If at any time, this module detects an error, it doesn't throw.
-// Instead, it stores an error report in the table key [ERROR].
-// Later, you can check for an error string using the function
-// final_check or final_check_throw.  If an error is
-// detected when there is already an error report in the table,
-// the error report is not overwritten, so therefore, the error
-// reported is always the first error detected.
-//
-// When this module finds a keyword in the table, it adds the keyword
-// to the [FOUND] set of all keywords successfully parsed.
-//
-// If the keyword table that you pass in isn't a table at all,
-// then the keyword-fetching functions will always return false.
-// Later, when you call 'check', an appropriate error will be
-// generated.
-//
-// The lua module 'keywords' contains the same functions as this
-// C++ class.  You can write code where a C++ function does some
-// of the parsing, and the lua code does the rest of the parsing.
-//
-////////////////////////////////////////////////////////////////////
-
-class LuaKeywordParser {
-private:
-    LuaVar found, error, key, val;
-    LuaSpecial keytab;
-    LuaCoreStack LS;
-    bool istable;
-
-    void init(const lua_State *L, int slot);
-public:
-    LuaKeywordParser(const LuaCoreStack &LS, LuaSlot slot);
-
-    // Fetch the value of the keyword.  If the keyword is found, then the
-    // keyword is added to the [FOUND] set, the value is returned in slot,
-    // and returns true.  Otherwise, sets slot to nil and returns false.
-    bool optional(LuaSlot slot, std::string_view kw);
-
-    // Fetch the value of the keyword.  If the keyword is found, then the
-    // keyword is added to the [FOUND] set, the value is returned in slot,
-    // and returns true.  Otherwise, sets slot to nil, returns false, and
-    // stores an [ERROR] report in the keyword table.
-    bool required(LuaSlot slot, std::string_view kw);
-
-    // Check if there are any errors so far, by checking for an [ERROR]
-    // report in the keyword table.  If any error has been
-    // detected, returns an error message, otherwise, returns empty
-    // string.
-    eng::string check();
-
-    // Check if there are any errors so far, by checking for an [ERROR]
-    // report in the keyword table.  Also check that all keyword
-    // arguments present in the table are in the [FOUND] set.  If there are
-    // any errors, returns an error message, otherwise returns empty string.
-    eng::string final_check();
-
-    // If check() returns an error, throws the error using luaL_error.
-    void check_throw();
-
-    // If final_check() returns an error, throws the error using luaL_error.
-    void final_check_throw();
-
-    // Fetch the state pointer. 
-    lua_State *state() const { return LS.state(); }
-};
 
 
 
-
-////////////////////////////////////////////////////////////////////
-//
-// Lua Byte Reader
-//
-// Converts a block of bytes in RAM into a lua_reader.
-//
-////////////////////////////////////////////////////////////////////
-
-class LuaByteReader {
-private:
-    const char *data_;
-    int64_t size_;
-public:
-    LuaByteReader(const char *d, int64_t s) : data_(d), size_(s) {}
-    void *lua_reader_userdata() { return this; }
-    static const char *lua_reader(lua_State *L, void *ud, size_t *size);
-};
-
 ////////////////////////////////////////////////////////////////////
 //
 // The Lua Constant Registry

luprex/cpp/core/planemap.hpp

@@ -79,6 +79,7 @@
 #include "wrap-bytell-hash-map.hpp"
 #include "util.hpp"
 #include "luastack.hpp"
+#include "keywords.hpp"
 
 #include <cstdint>
 #include <memory>

luprex/cpp/core/util.cpp

@@ -850,6 +850,18 @@ static std::string_view read_number_x(const char *p, bool plus, bool minus, bool
     return sv::read_number(source, plus, minus, dec, exp);
 }
 
+eng::string util::decode_token(uint64_t value) {
+    static const char encoding[] =
+        "\0_0123456789abcdefghijklmnopqrstuvwxyz";
+    uint64_t n = value;
+    char buffer[13] = {};
+    for (int i = 11; i >= 0; i--) {
+        buffer[i] = encoding[n % 38];
+        n /= 38;
+    }
+    return eng::string(buffer);
+}
+
 LuaDefine(unittests_util, "", "some unit tests") {
     // Test valid_hostname
     LuaAssert(L, sv::valid_hostname("foo123"));

luprex/cpp/core/util.hpp

@@ -497,6 +497,45 @@ constexpr auto hex32 = FormattedNumber<int>(0, true, 8, '0', 6);
 constexpr auto hex64 = FormattedNumber<int>(0, true, 16, '0', 6);
 constexpr auto dec = FormattedNumber<int>(0, false, 0, ' ', 6);
 
+// Encode a string as a token (fixed-width base38 number).
+// Each character is mapped to a digit 1-37 (0 means "no character"),
+// and the result is: CH0*38^11 + CH1*38^10 + ... + CH11*38^0.
+// Digit mapping: _ → 1, 0-9 → 2-11, a-z → 12-37.
+// Returns zero if the string is empty, too long, or contains
+// invalid characters.
+//
+static constexpr uint64_t encode_token(std::string_view str) {
+    if (str.size() > 12) return 0;
+    if (str.empty()) return 0;
+
+    uint64_t result = 0;
+    for (int i = 0; i < int(str.size()); i++) {
+        char c = str[i];
+        uint64_t digit = 0;
+        if (c == '_') {
+            digit = 1;
+        } else if ((c >= '0') && (c <= '9')) {
+            digit = uint64_t(c - '0') + 2;
+        } else if ((c >= 'a') && (c <= 'z')) {
+            digit = uint64_t(c - 'a') + 12;
+        } else if ((c >= 'A') && (c <= 'Z')) {
+            digit = uint64_t(c - 'A') + 12;
+        } else {
+            return 0;
+        }
+        result = result * 38 + digit;
+    }
+    // Pad remaining positions with zeros (no character).
+    for (int i = int(str.size()); i < 12; i++) {
+        result = result * 38;
+    }
+    return result;
+}
+
+// Decode a token (base38 number) back to a string.
+//
+eng::string decode_token(uint64_t value);
+
 } // namespace util
 
 template<class VALUE>