Add stuff that doesn't work

1 files changed, 26 insertions, 19 deletions

diff --git a/luatokens.lua b/luatokens.lua
index ecc3b6d..9ca19ac 100644
--- a/luatokens.lua
+++ b/luatokens.lua
@@ -272,7 +272,7 @@ do local tstring = selfify({})
             end
             return "string"
         end
-        tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring})
+        tsescapes.digitc = setmetatable(selfify({[parser.FALLBACK] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring})
         tsescapes.digitc[1]=function(state, token, rule)
             if rule == nil then
                 collect_fallback(state, string.char(state.in_digit))
@@ -346,7 +346,7 @@ do local tstring = selfify({})
         do local tseskipwhitespace = selfify(mknewline({
                 string = defs.string,
                 whitespace = "self",
-                [""] = "string",
+                [parser.FALLBACK] = "string",
                 [1] = collect_fallback,
             }, 2))
             --tseskipwhitespace["\n"] = setmetatable({[2] = countline, ["\r"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
@@ -366,7 +366,7 @@ do local tstring = selfify({})
     tstring['\n'] = false
     tstring['\r'] = false
 
-    tstring[""] = "self"
+    tstring[parser.FALLBACK] = "self"
 
     tstring[1] = collect_fallback
 
@@ -385,7 +385,7 @@ end
 
 do local tlongstring = {}
     defs.longstring = tlongstring
-    do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
+    do local tllongstring_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
         tllongstring_proper[1] = false -- placeholder for newline handling
         tllongstring_proper[2] = collect_fallback
 
@@ -412,7 +412,7 @@ do local tlongstring = {}
                     return "maybe_end"
                 end
             end
-            tllmaybe_end[""] = "longstring_proper"
+            tllmaybe_end[parser.FALLBACK] = "longstring_proper"
             tllmaybe_end[1] = collect_fallback
             tllmaybe_end[-1] = function(state, token, rule)
                 if not rule then
@@ -473,6 +473,7 @@ mknewline(defs, 1)
 defs["-"] = "maybe_comment"
 do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
     defs.maybe_comment = tmaybe_comment
+    tmaybe_comment[parser.EOZ] = "self" -- defs
     tmaybe_comment[-1] = function(state, token, rule)
         if rule ~= "comment" then
             state[#state+1] = "-"
@@ -480,12 +481,12 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
     end
     do local tmcomment = {comment_proper = selfify({})}
         tmaybe_comment.comment = tmcomment
-        tmcomment[""] = "comment_proper"
+        tmcomment[parser.FALLBACK] = "comment_proper"
         tmcomment["["] = "maybe_longcomment"
         mknewline(tmcomment, 1, defs)
         mknewline(tmcomment.comment_proper, 1, defs)
-        tmcomment.comment_proper[""] = "self"
-        do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
+        tmcomment.comment_proper[parser.FALLBACK] = "self"
+        do local tllongcomment_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
             tmcomment.longcomment = tllongcomment_proper
             do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
                 tllongcomment_proper.maybe_end = tllmaybe_end
@@ -504,7 +505,7 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
                         return "maybe_end"
                     end
                 end
-                tllmaybe_end[""] = "longcomment_proper"
+                tllmaybe_end[parser.FALLBACK] = "longcomment_proper"
                 tllmaybe_end[-1] = function(state, token, rule)
                     if not rule then
                         state.longcomment_close = nil
@@ -543,6 +544,7 @@ end
 local STATE = parser.STATE
 
 defs.multitokens = setmetatable({
+    [parser.EOZ] = "self",
     [-1] = function(state, token, rule)
         if not state[STATE].multitoken[token] then
             state[#state+1] = state[STATE].first
@@ -736,18 +738,23 @@ function defs.string_open(state, token)
     assert("this shouldn't happen")
 end
 
+local tokens = {
+    TK_AND = TK_AND, TK_BREAK = TK_BREAK,
+    TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION,
+    TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT,
+    TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE,
+    TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE,
+    TK_SHL = TK_SHL, TK_SHR = TK_SHR,
+    TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS,
+    TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING
+}
+for k,v in pairs(tokens) do
+    setmetatable(v, {__name=k, __tostring=function(self) return getmetatable(self).__name end})
+end
+
 return {
     defs = defs,
-    tokens = {
-        TK_AND = TK_AND, TK_BREAK = TK_BREAK,
-        TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION,
-        TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT,
-        TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE,
-        TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE,
-        TK_SHL = TK_SHL, TK_SHR = TK_SHR,
-        TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS,
-        TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING
-    },
+    tokens = tokens,
     reverse_keywords = reverse_keywords,
     reverse_tokens = {
         [TK_IDIV] = "//", [TK_CONCAT] = "..", [TK_DOTS] = "...", [TK_EQ] = "==", [TK_GE] = ">=", [TK_LE] = "<=", [TK_NE] = "~=",