Add stuff that doesn't work

1 files changed, 470 insertions, 0 deletions

diff --git a/testp.lua b/testp.lua
new file mode 100644
index 0000000..44bd5a3
--- /dev/null
+++ b/testp.lua
@@ -0,0 +1,470 @@
+--[[
+    This file is part of parser.lua - table based parsing
+    Copyright (C) 2019  Soni L.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+--]]
+
+local parser = require "parser"
+
+local caseno = 0
+local function case()
+    caseno = caseno + 1
+    return caseno
+end
+
+do -- basic check
+    local case = case()
+    local defs = {}
+    local count = 0
+    local state, err = parser.parse(defs, function() assert(count == 0, "should be called only once"); count = count + 1 return nil end)
+    assert(state)
+end -- basic check
+
+do -- trim left spaces
+    local defs = {}
+    defs.self = defs
+    defs[' '] = "whitespace"
+    defs['\n'] = "whitespace"
+    defs['\r'] = "whitespace"
+    defs['\t'] = "whitespace"
+    defs['\f'] = "whitespace"
+    defs['\v'] = "whitespace"
+    defs.whitespace = "self"
+    defs[parser.FALLBACK] = function(state, token)
+        state[#state + 1] = token
+        if #state > 20 then
+            state[1] = table.concat(state)
+            for i=#state, 2, -1 do
+                state[i] = nil
+            end
+        end
+        return "start"
+    end
+    defs.start = {}
+    defs.start.self = defs.start
+    defs.start[parser.FALLBACK] = function(state, token)
+        state[#state + 1] = token
+        if #state > 20 then
+            state[1] = table.concat(state)
+            for i=#state, 2, -1 do
+                state[i] = nil
+            end
+        end
+        return "self"
+    end
+    for k,v in ipairs({"hello", "    hello", "\t \v \n\r hello"}) do
+        local state, err = parser.parse(defs, v)
+        local case = case()
+        if not state then
+            print(case, err)
+        else
+            assert(table.concat(state) == "hello")
+        end
+    end
+end -- trim left spaces
+
+do -- lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [["hello world"]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(state[1] == luatokens.tokens.TK_STRING)
+        assert(state[2] == "hello world")
+        assert(state[3] == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- lua tokens
+
+do -- more lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [["\a\b\f\n\r\t\v\\\"\'\z        \x41\65\
+"]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(state[1] == luatokens.tokens.TK_STRING)
+        assert(state[2] == "\7\8\12\10\13\9\11\92\34\39\65\65\10")
+        assert(state[3] == nil)
+        assert(state.line == 2)
+    end
+end -- lua tokens
+
+do -- even more lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [["\u{000000000000000000000000000000000000000000000000000000000000041}"]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(state[1] == luatokens.tokens.TK_STRING)
+        assert(state[2] == "A")
+        assert(state[3] == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- lua tokens
+
+do -- even more lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [["\u{7F}""\u{80}""\u{7FF}""\u{800}""\u{FFFF}""\u{10000}""\u{1FFFFF}""\u{200000}""\u{3FFFFFF}""\u{4000000}""\u{7FFFFFFF}"]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\127")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\194\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\223\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\224\160\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\239\191\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\240\144\128\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\247\191\191\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\248\136\128\128\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\251\191\191\191\191")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\252\132\128\128\128\128")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "\253\191\191\191\191\191")
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- lua tokens
+
+do -- simple lua tokens
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [[[""]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == "[")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "")
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- lua tokens
+
+do -- simple long string
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [=[[[]]]=])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "")
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- long string
+
+do -- long string with depth 1
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [==[[=[]=]]==])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "")
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- long string
+
+do -- long string with "nested" long string
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [==[[=[[[]]]=]]==])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "[[]]")
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 1 or not state.line)
+    end
+end -- long string
+
+do -- long string edge cases
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=][[\n]][[\n ]]")
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "]=")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "]")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "] ")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == " ")
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 3 or not state.line)
+    end
+end -- long string
+
+do -- keywords
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [[
+     and       break     do        else      elseif    end
+     false     for       function  goto      if        in
+     local     nil       not       or        repeat    return
+     then      true      until     while]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == luatokens.tokens.TK_AND)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_BREAK)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_DO)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_ELSE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_ELSEIF)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_END)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FALSE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FOR)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FUNCTION)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_GOTO)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_IF)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_IN)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_LOCAL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_NIL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_NOT)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_OR)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_REPEAT)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_RETURN)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_THEN)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_TRUE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_UNTIL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_WHILE)
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 4)
+    end
+end -- keywords
+
+do -- "other tokens"
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [[
+     +     -     *     /     %     ^     #
+     &     ~     |     <<    >>    //
+     ==    ~=    <=    >=    <     >     =
+     (     )     {     }     [     ]     ::
+     ;     :     ,     .     ..    ...]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == "+")
+        assert(table.remove(state, 1) == "-")
+        assert(table.remove(state, 1) == "*")
+        assert(table.remove(state, 1) == "/")
+        assert(table.remove(state, 1) == "%")
+        assert(table.remove(state, 1) == "^")
+        assert(table.remove(state, 1) == "#")
+        assert(table.remove(state, 1) == "&")
+        assert(table.remove(state, 1) == "~")
+        assert(table.remove(state, 1) == "|")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_SHL)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_SHR)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_IDIV)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_EQ)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_NE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_LE)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_GE)
+        assert(table.remove(state, 1) == "<")
+        assert(table.remove(state, 1) == ">")
+        assert(table.remove(state, 1) == "=")
+        assert(table.remove(state, 1) == "(")
+        assert(table.remove(state, 1) == ")")
+        assert(table.remove(state, 1) == "{")
+        assert(table.remove(state, 1) == "}")
+        assert(table.remove(state, 1) == "[")
+        assert(table.remove(state, 1) == "]")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_DBCOLON)
+        assert(table.remove(state, 1) == ";")
+        assert(table.remove(state, 1) == ":")
+        assert(table.remove(state, 1) == ",")
+        assert(table.remove(state, 1) == ".")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_CONCAT)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_DOTS)
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 5)
+    end
+end -- "other tokens"
+
+do -- long comments
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [==[--[[
+      
+    --]]]==])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 3)
+    end
+end -- long comments
+
+do -- numbers
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [[
+     3   345   0xff   0xBEBADA
+     3.0     3.1416     314.16e-2     0.31416E1     34e1
+     0x0.1E  0xA23p-4   0X1.921FB54442D18P+1]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        -- integers
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 3)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 345)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 0xff)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 0xBEBADA)
+        -- floats
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 3.0)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 3.1416)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 314.16e-2)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 0.31416E1)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 34e1)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 0.1171875)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 162.1875)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == math.pi)
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 3)
+    end
+end -- numbers
+
+do -- FUCK
+    local luatokens = require "luatokens"
+    local luatokens_file = io.open("./luatokens.lua", "r")
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, function() return luatokens_file:read(8192) end)
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    end
+end -- FUCK