Keywords and long strings

1 files changed, 17 insertions, 3 deletions

diff --git a/parser.lua b/parser.lua
index bfa7dd3..4f4e166 100644
--- a/parser.lua
+++ b/parser.lua
@@ -24,6 +24,8 @@ local DATA = {}
 local GEN = {}
 -- key for DATA OFFSET
 local OFFDATA = {}
+-- key for End of Stream
+local EOZ = {}
 
 local optimize_lookups = {}
 for i=0, 255 do
@@ -39,6 +41,9 @@ local function get_next_common(state, in_pos, token)
     if state[STATE] then
         local st = state[STATE]
         local rule = st[token]
+        if not rule and token == EOZ then
+            return in_pos, state
+        end
         do -- pre-hooks
             local pos = -1
             local hook = st[pos]
@@ -83,7 +88,9 @@ local function get_next_common(state, in_pos, token)
 end
 
 local function get_next_table(state, in_pos)
-    if state[DATA] == nil or #state[DATA] == 0 then return in_pos, state end -- TODO end-of-stream handling
+    if state[DATA] == nil or #state[DATA] == 0 then
+        return get_next_common(state, in_pos, EOZ)
+    end
     in_pos = in_pos + 1
     local token = state[DATA][in_pos - state[OFFDATA]]
     if token == nil then
@@ -95,7 +102,13 @@ local function get_next_table(state, in_pos)
 end
 
 local function get_next_string(state, in_pos)
-    if state[DATA] == nil or #state[DATA] == 0 then return in_pos, state end -- TODO end-of-stream handling
+    if state[DATA] == nil or #state[DATA] == 0 then
+        if state[STATE] == nil then
+            return in_pos, state
+        else
+            return get_next_common(state, in_pos, EOZ)
+        end
+    end
     in_pos = in_pos + 1
     local token = optimize_lookups[string.byte(state[DATA], in_pos - state[OFFDATA], in_pos - state[OFFDATA])]
     if token == nil then
@@ -142,6 +155,7 @@ local COLLECT = {}
 return {
     STATE = STATE,
     COLLECT = COLLECT,
+    EOZ = EOZ,
     stream = stream,
     parse = parse,
     -- common utility function
@@ -154,7 +168,7 @@ return {
         if not rule then
             local t = state[COLLECT]
             t[#t+1] = token
-            if t.coalesce and #t > t.coalesce then
+            if t.coalesce and #t >= t.coalesce then
                 t[1] = table.concat(t)
                 for i=2, #t do t[i] = nil end
             end