Attempted lua tokenizer didn't work

Publishing anyway because someone might be able to learn from my failure

1 files changed, 28 insertions, 8 deletions

diff --git a/parser.lua b/parser.lua
index 479d80a..ece8a8f 100644
--- a/parser.lua
+++ b/parser.lua
@@ -31,27 +31,34 @@ local type, tostring
 local function get_next_common(state, in_pos, token)
     -- note: must preserve "token" - do not call recursively with a different token
     local transition
-    if state[STATE] ~= nil then
-        transition = state[STATE][token]
-        if not transition then
-            transition = state[STATE][""]
+    if state[STATE] then
+        local st = state[STATE]
+        local rule = st[token]
+        transition = rule
+        if transition == nil then
+            transition = st[""]
         end
         local recheck = true
         while recheck do
             recheck = false
             local tytrans = type(transition)
             if tytrans == "string" then
-                transition = state[STATE][transition]
+                transition = st[transition]
                 recheck = true
             elseif tytrans == "function" then
                 transition = transition(state, token)
                 recheck = true
             end
         end
-        state[STATE] = transition -- may be nil
+        for i, hook in ipairs(st) do
+            if hook then -- allow overriding/disabling hooks
+                hook(state, token, rule)
+            end
+        end
+        state[STATE] = transition -- may be nil or false
     end
-    -- must NOT use elseif here - the above may set state to nil!
-    if state[STATE] == nil then
+    -- must NOT use elseif here - the above may set state to nil or false!
+    if not state[STATE] then
         -- unexpected token. stream consumer may attempt to recover,
         -- but we do this mostly to differentiate it from "end of stream" condition.
         return in_pos - 1, nil, "unexpected token", token, state
@@ -112,8 +119,21 @@ local function parse(defs, data)
     end
 end
 
+-- utility function that's quite common
+local function selfify(t)
+    t.self = t
+    return t
+end
+-- common hook
+local function insert_fallback(state, token, rule)
+    if not rule then
+        state[#state+1] = token
+    end
+end
+
 return {
     STATE = STATE,
     stream = stream,
     parse = parse,
+    selfify = selfify,
 }