diff options
Diffstat (limited to 'testp.lua')
| -rw-r--r-- | testp.lua | 470 |
1 files changed, 470 insertions, 0 deletions
diff --git a/testp.lua b/testp.lua new file mode 100644 index 0000000..44bd5a3 --- /dev/null +++ b/testp.lua @@ -0,0 +1,470 @@ +--[[ + This file is part of parser.lua - table based parsing + Copyright (C) 2019 Soni L. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +--]] + +local parser = require "parser" + +local caseno = 0 +local function case() + caseno = caseno + 1 + return caseno +end + +do -- basic check + local case = case() + local defs = {} + local count = 0 + local state, err = parser.parse(defs, function() assert(count == 0, "should be called only once"); count = count + 1 return nil end) + assert(state) +end -- basic check + +do -- trim left spaces + local defs = {} + defs.self = defs + defs[' '] = "whitespace" + defs['\n'] = "whitespace" + defs['\r'] = "whitespace" + defs['\t'] = "whitespace" + defs['\f'] = "whitespace" + defs['\v'] = "whitespace" + defs.whitespace = "self" + defs[parser.FALLBACK] = function(state, token) + state[#state + 1] = token + if #state > 20 then + state[1] = table.concat(state) + for i=#state, 2, -1 do + state[i] = nil + end + end + return "start" + end + defs.start = {} + defs.start.self = defs.start + defs.start[parser.FALLBACK] = function(state, token) + state[#state + 1] = token + if #state > 20 then + state[1] = table.concat(state) + for i=#state, 2, -1 do + state[i] = nil + end + end + return "self" + end + for k,v in ipairs({"hello", " hello", "\t \v \n\r hello"}) do + local state, err = parser.parse(defs, v) + local case = case() + if not state then + print(case, err) + else + assert(table.concat(state) == "hello") + end + end +end -- trim left spaces + +do -- lua tokens + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [["hello world"]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(state[1] == luatokens.tokens.TK_STRING) + assert(state[2] == "hello world") + assert(state[3] == nil) + assert(state.line == 1 or not state.line) + end +end -- lua tokens + +do -- more lua tokens + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [["\a\b\f\n\r\t\v\\\"\'\z \x41\65\ +"]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(state[1] == luatokens.tokens.TK_STRING) + assert(state[2] == "\7\8\12\10\13\9\11\92\34\39\65\65\10") + assert(state[3] == nil) + assert(state.line == 2) + end +end -- lua tokens + +do -- even more lua tokens + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [["\u{000000000000000000000000000000000000000000000000000000000000041}"]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(state[1] == luatokens.tokens.TK_STRING) + assert(state[2] == "A") + assert(state[3] == nil) + assert(state.line == 1 or not state.line) + end +end -- lua tokens + +do -- even more lua tokens + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [["\u{7F}""\u{80}""\u{7FF}""\u{800}""\u{FFFF}""\u{10000}""\u{1FFFFF}""\u{200000}""\u{3FFFFFF}""\u{4000000}""\u{7FFFFFFF}"]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\127") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\194\128") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\223\191") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\224\160\128") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\239\191\191") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\240\144\128\128") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\247\191\191\191") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\248\136\128\128\128") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\251\191\191\191\191") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\252\132\128\128\128\128") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "\253\191\191\191\191\191") + assert(table.remove(state, 1) == nil) + assert(state.line == 1 or not state.line) + end +end -- lua tokens + +do -- simple lua tokens + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [[[""]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == "[") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "") + assert(table.remove(state, 1) == nil) + assert(state.line == 1 or not state.line) + end +end -- lua tokens + +do -- simple long string + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [=[[[]]]=]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "") + assert(table.remove(state, 1) == nil) + assert(state.line == 1 or not state.line) + end +end -- long string + +do -- long string with depth 1 + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [==[[=[]=]]==]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "") + assert(table.remove(state, 1) == nil) + assert(state.line == 1 or not state.line) + end +end -- long string + +do -- long string with "nested" long string + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [==[[=[[[]]]=]]==]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "[[]]") + assert(table.remove(state, 1) == nil) + assert(state.line == 1 or not state.line) + end +end -- long string + +do -- long string edge cases + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=][[\n]][[\n ]]") + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "]=") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "]") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "] ") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == "") + assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) + assert(table.remove(state, 1) == " ") + assert(table.remove(state, 1) == nil) + assert(state.line == 3 or not state.line) + end +end -- long string + +do -- keywords + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [[ + and break do else elseif end + false for function goto if in + local nil not or repeat return + then true until while]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == luatokens.tokens.TK_AND) + assert(table.remove(state, 1) == luatokens.tokens.TK_BREAK) + assert(table.remove(state, 1) == luatokens.tokens.TK_DO) + assert(table.remove(state, 1) == luatokens.tokens.TK_ELSE) + assert(table.remove(state, 1) == luatokens.tokens.TK_ELSEIF) + assert(table.remove(state, 1) == luatokens.tokens.TK_END) + assert(table.remove(state, 1) == luatokens.tokens.TK_FALSE) + assert(table.remove(state, 1) == luatokens.tokens.TK_FOR) + assert(table.remove(state, 1) == luatokens.tokens.TK_FUNCTION) + assert(table.remove(state, 1) == luatokens.tokens.TK_GOTO) + assert(table.remove(state, 1) == luatokens.tokens.TK_IF) + assert(table.remove(state, 1) == luatokens.tokens.TK_IN) + assert(table.remove(state, 1) == luatokens.tokens.TK_LOCAL) + assert(table.remove(state, 1) == luatokens.tokens.TK_NIL) + assert(table.remove(state, 1) == luatokens.tokens.TK_NOT) + assert(table.remove(state, 1) == luatokens.tokens.TK_OR) + assert(table.remove(state, 1) == luatokens.tokens.TK_REPEAT) + assert(table.remove(state, 1) == luatokens.tokens.TK_RETURN) + assert(table.remove(state, 1) == luatokens.tokens.TK_THEN) + assert(table.remove(state, 1) == luatokens.tokens.TK_TRUE) + assert(table.remove(state, 1) == luatokens.tokens.TK_UNTIL) + assert(table.remove(state, 1) == luatokens.tokens.TK_WHILE) + assert(table.remove(state, 1) == nil) + assert(state.line == 4) + end +end -- keywords + +do -- "other tokens" + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [[ + + - * / % ^ # + & ~ | << >> // + == ~= <= >= < > = + ( ) { } [ ] :: + ; : , . .. ...]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == "+") + assert(table.remove(state, 1) == "-") + assert(table.remove(state, 1) == "*") + assert(table.remove(state, 1) == "/") + assert(table.remove(state, 1) == "%") + assert(table.remove(state, 1) == "^") + assert(table.remove(state, 1) == "#") + assert(table.remove(state, 1) == "&") + assert(table.remove(state, 1) == "~") + assert(table.remove(state, 1) == "|") + assert(table.remove(state, 1) == luatokens.tokens.TK_SHL) + assert(table.remove(state, 1) == luatokens.tokens.TK_SHR) + assert(table.remove(state, 1) == luatokens.tokens.TK_IDIV) + assert(table.remove(state, 1) == luatokens.tokens.TK_EQ) + assert(table.remove(state, 1) == luatokens.tokens.TK_NE) + assert(table.remove(state, 1) == luatokens.tokens.TK_LE) + assert(table.remove(state, 1) == luatokens.tokens.TK_GE) + assert(table.remove(state, 1) == "<") + assert(table.remove(state, 1) == ">") + assert(table.remove(state, 1) == "=") + assert(table.remove(state, 1) == "(") + assert(table.remove(state, 1) == ")") + assert(table.remove(state, 1) == "{") + assert(table.remove(state, 1) == "}") + assert(table.remove(state, 1) == "[") + assert(table.remove(state, 1) == "]") + assert(table.remove(state, 1) == luatokens.tokens.TK_DBCOLON) + assert(table.remove(state, 1) == ";") + assert(table.remove(state, 1) == ":") + assert(table.remove(state, 1) == ",") + assert(table.remove(state, 1) == ".") + assert(table.remove(state, 1) == luatokens.tokens.TK_CONCAT) + assert(table.remove(state, 1) == luatokens.tokens.TK_DOTS) + assert(table.remove(state, 1) == nil) + assert(state.line == 5) + end +end -- "other tokens" + +do -- long comments + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [==[--[[ + + --]]]==]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + assert(table.remove(state, 1) == nil) + assert(state.line == 3) + end +end -- long comments + +do -- numbers + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [[ + 3 345 0xff 0xBEBADA + 3.0 3.1416 314.16e-2 0.31416E1 34e1 + 0x0.1E 0xA23p-4 0X1.921FB54442D18P+1]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + -- integers + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 3) + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 345) + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 0xff) + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 0xBEBADA) + -- floats + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 3.0) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 3.1416) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 314.16e-2) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 0.31416E1) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 34e1) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 0.1171875) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 162.1875) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == math.pi) + assert(table.remove(state, 1) == nil) + assert(state.line == 3) + end +end -- numbers + +do -- FUCK + local luatokens = require "luatokens" + local luatokens_file = io.open("./luatokens.lua", "r") + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, function() return luatokens_file:read(8192) end) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + end +end -- FUCK |