Keywords and long strings

1 files changed, 99 insertions, 3 deletions

diff --git a/test.lua b/test.lua
index 8672903..ef0a586 100644
--- a/test.lua
+++ b/test.lua
@@ -24,6 +24,14 @@ local function case()
     return caseno
 end
 
+do -- basic check
+    local case = case()
+    local defs = {}
+    local count = 0
+    local state, err = parser.parse(defs, function() assert(count == 0, "should be called only once"); count = count + 1 return nil end)
+    assert(state)
+end -- basic check
+
 do -- trim left spaces
     local defs = {}
     defs.self = defs
@@ -82,6 +90,7 @@ do -- lua tokens
     else
         assert(state[1] == luatokens.tokens.TK_STRING)
         assert(state[2] == "hello world")
+        assert(state.line == 1 or not state.line)
     end
 end -- lua tokens
 
@@ -101,6 +110,7 @@ do -- more lua tokens
     else
         assert(state[1] == luatokens.tokens.TK_STRING)
         assert(state[2] == "\7\8\12\10\13\9\11\92\34\39\65\65\10")
+        assert(state.line == 2)
     end
 end -- lua tokens
 
@@ -119,6 +129,7 @@ do -- even more lua tokens
     else
         assert(state[1] == luatokens.tokens.TK_STRING)
         assert(state[2] == "A")
+        assert(state.line == 1 or not state.line)
     end
 end -- lua tokens
 
@@ -157,6 +168,7 @@ do -- even more lua tokens
         assert(table.remove(state, 1) == "\252\132\128\128\128\128")
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "\253\191\191\191\191\191")
+        assert(state.line == 1 or not state.line)
     end
 end -- lua tokens
 
@@ -176,6 +188,7 @@ do -- simple lua tokens
         assert(table.remove(state, 1) == "[")
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "")
+        assert(state.line == 1 or not state.line)
     end
 end -- lua tokens
 
@@ -194,8 +207,9 @@ do -- simple long string
     else
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "")
+        assert(state.line == 1 or not state.line)
     end
-end -- lua tokens
+end -- long string
 
 do -- long string with depth 1
     local luatokens = require "luatokens"
@@ -212,8 +226,9 @@ do -- long string with depth 1
     else
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "")
+        assert(state.line == 1 or not state.line)
     end
-end -- lua tokens
+end -- long string
 
 do -- long string with "nested" long string
     local luatokens = require "luatokens"
@@ -230,5 +245,86 @@ do -- long string with "nested" long string
     else
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "[[]]")
+        assert(state.line == 1 or not state.line)
     end
-end -- lua tokens
+end -- long string
+
+do -- long string edge cases
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==]")
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "]=")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "]")
+        assert(state.line == 1 or not state.line)
+    end
+end -- long string
+
+do -- keywords
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [[
+     and       break     do        else      elseif    end
+     false     for       function  goto      if        in
+     local     nil       not       or        repeat    return
+     then      true      until     while]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_AND)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_BREAK)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_DO)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_ELSE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_ELSEIF)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_END)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FALSE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FOR)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FUNCTION)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_GOTO)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_IF)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_IN)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_LOCAL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_NIL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_NOT)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_OR)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_REPEAT)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_RETURN)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_THEN)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_TRUE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_UNTIL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_WHILE)
+        assert(state.line == 4)
+    end
+end -- keywords
+
+do -- FUCK
+    local luatokens = require "luatokens"
+    local luatokens_file = io.open("./luatokens.lua", "r"):read((_VERSION == "5.1" or _VERSION == "5.2") and "*a" or "a")
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, luatokens_file)
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    end
+end -- FUCK