Attempted lua tokenizer didn't work

Publishing anyway because someone might be able to learn from my failure

1 files changed, 51 insertions, 3 deletions

diff --git a/test.lua b/test.lua
index 606d60e..1290c97 100644
--- a/test.lua
+++ b/test.lua
@@ -18,6 +18,12 @@
 
 local parser = require "parser"
 
+local caseno = 0
+local function case()
+    caseno = caseno + 1
+    return caseno
+end
+
 do -- trim left spaces
     local defs = {}
     defs.self = defs
@@ -52,7 +58,49 @@ do -- trim left spaces
     end
     for k,v in ipairs({"hello", "    hello", "\t \v \n\r hallo", "I really like this parser thingy if it can be called that"}) do
         local state, err = parser.parse(defs, v)
-        if not state then print(err) end
-        print(table.concat(state))
+        if not state then
+            print(case(), err)
+        else
+            print(case(), table.concat(state))
+        end
     end
-end
+end -- trim left spaces
+
+do -- lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.tokens
+    local state, err, etoken, estate = parser.parse(tokens, [["hello world"]])
+    local case = case()
+    print(case, "---- IN  TOKENS ----")
+    if not state then
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+    else
+        for i,v in ipairs(state) do
+            print(case, i, v)
+        end
+    end
+    print(case, "---- OUT TOKENS ----")
+end -- lua tokens
+
+do -- more lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.tokens
+    local state, err, etoken, estate = parser.parse(tokens, [["\a\b\f\n\r\t\v\\\"\'\z        \x41\65\
+"]])
+    local case = case()
+    print(case, "---- IN  TOKENS ----")
+    if not state then
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+    else
+        for i,v in ipairs(state) do
+            print(case, i, v)
+        end
+    end
+    print(case, "---- OUT TOKENS ----")
+end -- lua tokens