Fix \u

1 files changed, 68 insertions, 19 deletions

diff --git a/test.lua b/test.lua
index 076d1e8..283b566 100644
--- a/test.lua
+++ b/test.lua
@@ -56,57 +56,106 @@ do -- trim left spaces
         end
         return "self"
     end
-    for k,v in ipairs({"hello", "    hello", "\t \v \n\r hallo", "I really like this parser thingy if it can be called that"}) do
+    for k,v in ipairs({"hello", "    hello", "\t \v \n\r hello"}) do
         local state, err = parser.parse(defs, v)
+        local case = case()
         if not state then
-            print(case(), err)
+            print(case, err)
         else
-            print(case(), table.concat(state))
+            assert(table.concat(state) == "hello")
         end
     end
 end -- trim left spaces
 
 do -- lua tokens
     local luatokens = require "luatokens"
-    local tokens = luatokens.tokens
+    local tokens = luatokens.defs
     local state, err, etoken, estate = parser.parse(tokens, [["hello world"]])
     local case = case()
-    print(case, "---- IN  TOKENS ----")
     if not state then
+        print(case, "---- IN  TOKENS ----")
         print(case, err, etoken)
         for i,v in pairs(estate) do
             print(case, i, v)
         end
+        print(case, "---- OUT TOKENS ----")
     else
-        for i,v in ipairs(state) do
-            print(case, i, v)
-        end
+        assert(state[1] == luatokens.tokens.TK_STRING)
+        assert(state[2] == "hello world")
     end
-    print(case, "---- OUT TOKENS ----")
 end -- lua tokens
 
 do -- more lua tokens
     local luatokens = require "luatokens"
-    local tokens = luatokens.tokens
+    local tokens = luatokens.defs
     local state, err, etoken, estate = parser.parse(tokens, [["\a\b\f\n\r\t\v\\\"\'\z        \x41\65\
 "]])
     local case = case()
-    print(case, "---- IN  TOKENS ----")
     if not state then
+        print(case, "---- IN  TOKENS ----")
         print(case, err, etoken)
         for i,v in pairs(estate) do
             print(case, i, v)
         end
+        print(case, "---- OUT TOKENS ----")
     else
-        for i,v in ipairs(state) do
+        assert(state[1] == luatokens.tokens.TK_STRING)
+        assert(state[2] == "\7\8\12\10\13\9\11\92\34\39\65\65\10")
+    end
+end -- lua tokens
+
+do -- even more lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [["\u{000000000000000000000000000000000000000000000000000000000000041}"]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
             print(case, i, v)
-            if v == luatokens.TK_STRING then
-                in_string = true
-            elseif in_string then
-                print(case, v:gsub(".", function(v) return "\\"..string.byte(v) end))
-                in_string = false
-            end
         end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(state[1] == luatokens.tokens.TK_STRING)
+        assert(state[2] == "A")
+    end
+end -- lua tokens
+
+do -- even more lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [["\u{7F}""\u{80}""\u{7FF}""\u{800}""\u{FFFF}""\u{10000}""\u{1FFFFF}""\u{200000}""\u{3FFFFFF}""\u{4000000}""\u{7FFFFFFF}"]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\127")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\194\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\223\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\224\160\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\239\191\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\240\144\128\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\247\191\191\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\248\136\128\128\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\251\191\191\191\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\252\132\128\128\128\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\253\191\191\191\191\191")
     end
-    print(case, "---- OUT TOKENS ----")
 end -- lua tokens