diff options
| -rw-r--r-- | luatokens.lua | 82 | ||||
| -rw-r--r-- | printtokens.lua | 35 | ||||
| -rw-r--r-- | test.lua | 2 |
3 files changed, 111 insertions, 8 deletions
diff --git a/luatokens.lua b/luatokens.lua index af53d45..5217ab5 100644 --- a/luatokens.lua +++ b/luatokens.lua @@ -581,11 +581,14 @@ defs["."] = setmetatable({ if token ~= "." then if rule ~= "digit" then state[#state+1] = "." - else - error("NYI") -- TODO digit handling end end end, + digit = function(state, token, rule) + state[#state+1] = TK_FLT + state[COLLECT] = {".", coalesce=31} + return "in_decimal" + end, ["."] = setmetatable({ [-1] = function(state, token, rule) if token ~= "." then @@ -595,17 +598,82 @@ defs["."] = setmetatable({ ["."] = function(state, token) state[#state+1] = TK_DOTS return "self" -- actually goes into defs - end + end, }, {__index=defs}) }, {__index=defs}) function defs.digit(state, token) - -- TODO + state[COLLECT] = {token, coalesce=31} + if token == "0" then + return "in_zero" + else + return "in_integer" + end end -defs.in_digit = { - -- TODO -} +defs.in_integer = setmetatable(selfify({ + hexdigit = "alpha", + alpha = false, + ['e'] = "exp", + ['E'] = "exp", + exp = function(state, token) + collect_fallback(state, token) + return "in_exp" + end, + ['.'] = function(state, token) + collect_fallback(state, token) + return "in_decimal" + end, + digit = function(state, token) + collect_fallback(state, token) + return "in_digit" + end, +}, "in_digit"), {__index=defs}) + +defs.in_zero = setmetatable({ + ['x'] = function(state, token) + collect_fallback(state, token) + return "in_hex" + end, +}, {__index=defs.in_integer}) + +defs.in_decimal = setmetatable(selfify({ + ['.'] = false, +}, "in_digit"), {__index=defs.in_integer}) + +defs.in_expnum = setmetatable(selfify({ + exp = false, +}, "in_digit"), {__index=defs.in_decimal}) + +defs.in_subexp = setmetatable({ + in_expnum = defs.in_expnum, + digit = function(state, token) + collect_fallback(state, token) + return "in_expnum" + end, +}, {__index=defs.base}) + +defs.in_exp = setmetatable({ + ["+"] = "sign", + ["-"] = "sign", + sign = function(state, token) + collect_fallback(state, token) + return "in_subexp" + end, +}, {__index=defs.in_subexp}) + +defs.in_hex = setmetatable({ + in_decimal = "in_hex_fraction", + hexdigit = 'digit', + ['e'] = 'hexdigit', + ['E'] = 'hexdigit', + ['p'] = 'exp', + ['P'] = 'exp', +}, {__index=defs.in_integer}) + +defs.in_hex_fraction = setmetatable({ + ['.'] = false, +}, {__index=defs.in_hex}) function defs.simpletoken(state, token) state[#state+1] = token diff --git a/printtokens.lua b/printtokens.lua new file mode 100644 index 0000000..8f2d0f5 --- /dev/null +++ b/printtokens.lua @@ -0,0 +1,35 @@ +--[[ + This file is part of luatokens.lua - pure-Lua Lua tokenizer + Copyright (C) 2019 Soni L. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +--]] + +local parser = require "parser" + +do + local filename = arg[1] + local luatokens = require "luatokens" + local file = io.open(filename, "r") + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, function() return file:read(8192) end) + if state then + for i,v in ipairs(state) do + v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v + print(i, v) -- TODO formatting + end + else + print("Parse error") + end +end diff --git a/test.lua b/test.lua index b0b7082..077f139 100644 --- a/test.lua +++ b/test.lua @@ -405,7 +405,7 @@ do -- long comments end end -- long comments -while false do -- FUCK +do -- FUCK local luatokens = require "luatokens" local luatokens_file = io.open("./luatokens.lua", "r") local tokens = luatokens.defs |