talezero/integration
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add _ to the set of characters allowed in tokens

Browse Source
This commit is contained in:
jyelon
2026-02-19 23:35:38 -05:00
parent 3f2f3416c6
commit d79ecef1fe
5 changed files with 38 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
luprex/cpp/core/luastack.cpp
View File

@@ -47,16 +47,13 @@ LuaConstantReg *LuaConstantReg::All;
eng::string LuaToken::str() const {
static const char encoding[] =
"\0_0123456789abcdefghijklmnopqrstuvwxyz";
uint64_t n = (uint64_t)value;
char buffer[13] = {};
for (int i = 11; i >= 0; i--) {
int d = n % 37;
n /= 37;
if (d >= 1 && d <= 10) {
buffer[i] = '0' + (d - 1);
} else if (d >= 11 && d <= 36) {
buffer[i] = 'a' + (d - 11);
}
buffer[i] = encoding[n % 38];
n /= 38;
}
return eng::string(buffer);
}
@@ -974,6 +971,9 @@ LuaDefine(unittests_token, "", "Unit tests for LuaToken encoding") {
LuaAssertStrEq(L, LuaToken("a0").str(), "a0");
LuaAssertStrEq(L, LuaToken("0a").str(), "0a");
LuaAssertStrEq(L, LuaToken("000000000000").str(), "000000000000");
LuaAssertStrEq(L, LuaToken("foo_bar").str(), "foo_bar");
LuaAssertStrEq(L, LuaToken("_").str(), "_");
LuaAssertStrEq(L, LuaToken("a_b").str(), "a_b");
// Test that empty/invalid strings produce the empty token.
LuaAssert(L, LuaToken(std::string_view("")).empty());
@@ -990,6 +990,10 @@ LuaDefine(unittests_token, "", "Unit tests for LuaToken encoding") {
LuaAssert(L, LuaToken("hello").value > LuaToken("hell").value);
LuaAssert(L, LuaToken("a0").value > LuaToken("a").value);
LuaAssert(L, LuaToken("a").value != LuaToken("a0").value);
LuaAssert(L, LuaToken("0").value > LuaToken("_").value);
LuaAssert(L, LuaToken("a").value > LuaToken("9").value);
LuaAssert(L, LuaToken("a_b").value > LuaToken("a_a").value);
LuaAssert(L, LuaToken("foo_bar").value != LuaToken("foobar").value);
return 0;
}