Add token literals to the lua parser

luprex/ext/eris-master/src/llex.c

@@ -39,7 +39,7 @@ static const char *const luaX_tokens [] = {
     "in", "local", "nil", "not", "or", "repeat",
     "return", "then", "true", "until", "while",
     "..", "...", "==", ">=", "<=", "~=", "::", "<eof>",
-    "<number>", "<name>", "<string>"
+    "<number>", "<name>", "<string>", "<token>"
 };
 
 
@@ -93,6 +93,7 @@ static const char *txtToken (LexState *ls, int token) {
     case TK_NAME:
     case TK_STRING:
     case TK_NUMBER:
+    case TK_TOKEN:
       save(ls, '\0');
       return luaO_pushfstring(ls->L, LUA_QS, luaZ_buffer(ls->buff));
     default:
@@ -485,6 +486,28 @@ static int llex (LexState *ls, SemInfo *seminfo) {
       case EOZ: {
         return TK_EOS;
       }
+      case '@': {  /* token literal */
+        size_t tokval = 0;
+        int toklen = 0;
+        save_and_next(ls);
+        while (1) {
+          char c = (char)ls->current;
+          size_t digit;
+          if (c >= '0' && c <= '9') digit = (size_t)(c - '0') + 1;
+          else if (c >= 'a' && c <= 'z') digit = (size_t)(c - 'a') + 11;
+          else if (c >= 'A' && c <= 'Z') digit = (size_t)(c - 'A') + 11;
+          else break;
+          tokval = tokval * 37 + digit;
+          toklen++;
+          save_and_next(ls);
+        }
+        if (toklen == 0 || toklen > 12 || ls->current == '_')
+          lexerror(ls, "invalid token literal", TK_TOKEN);
+        /* Pad to fixed width of 12 digits. */
+        for (int i = toklen; i < 12; i++) tokval *= 37;
+        seminfo->p = (void *)tokval;
+        return TK_TOKEN;
+      }
       default: {
         if (lislalpha(ls->current)) {  /* identifier or reserved word? */
           TString *ts;