It's working better now...

1 files changed, 22 insertions, 13 deletions

diff --git a/parser.lua b/parser.lua
index ece8a8f..ff8378c 100644
--- a/parser.lua
+++ b/parser.lua
@@ -25,6 +25,11 @@ local GEN = {}
 -- key for DATA OFFSET
 local OFFDATA = {}
 
+local optimize_lookups = {}
+for i=0, 255 do
+    optimize_lookups[i] = string.char(i)
+end
+
 local type, tostring
     = type, tostring
 
@@ -81,7 +86,7 @@ end
 local function get_next_string(state, in_pos)
     if state[DATA] == nil or #state[DATA] == 0 then return in_pos, state end
     in_pos = in_pos + 1
-    local token = state[DATA]:sub(in_pos - state[OFFDATA], in_pos - state[OFFDATA])
+    local token = optimize_lookups[string.byte(state[DATA], in_pos - state[OFFDATA], in_pos - state[OFFDATA])] or ""
     if token == "" then
         state[OFFDATA] = in_pos - 1
         state[DATA] = state[GEN]()
@@ -119,21 +124,25 @@ local function parse(defs, data)
     end
 end
 
--- utility function that's quite common
-local function selfify(t)
-    t.self = t
-    return t
-end
--- common hook
-local function insert_fallback(state, token, rule)
-    if not rule then
-        state[#state+1] = token
-    end
-end
+-- not used by any of the above but useful for others
+
+local COLLECT = {}
 
 return {
     STATE = STATE,
+    COLLECT = COLLECT,
     stream = stream,
     parse = parse,
-    selfify = selfify,
+    -- common utility function
+    selfify = function(t)
+        t.self = t
+        return t
+    end,
+    -- common hook
+    collect_fallback = function(state, token, rule)
+        if not rule then
+            local t = state[COLLECT]
+            t[#t+1] = token
+        end
+    end,
 }