diff options
| -rw-r--r-- | luatokens.lua | 24 | ||||
| -rw-r--r-- | test.lua | 47 |
2 files changed, 66 insertions, 5 deletions
diff --git a/luatokens.lua b/luatokens.lua index 5217ab5..ecc3b6d 100644 --- a/luatokens.lua +++ b/luatokens.lua @@ -616,6 +616,7 @@ defs.in_integer = setmetatable(selfify({ alpha = false, ['e'] = "exp", ['E'] = "exp", + [parser.EOZ] = "self", -- defs exp = function(state, token) collect_fallback(state, token) return "in_exp" @@ -628,10 +629,20 @@ defs.in_integer = setmetatable(selfify({ collect_fallback(state, token) return "in_digit" end, + [-1] = function(state, token, rule) + -- TODO figure out best order for these checks + if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end + state[#state+1] = state[STATE].numtype + state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option + state[COLLECT] = nil + end, + numtype = TK_INT }, "in_digit"), {__index=defs}) defs.in_zero = setmetatable({ - ['x'] = function(state, token) + ['x'] = "into_hex", + ['X'] = "into_hex", + into_hex = function(state, token) collect_fallback(state, token) return "in_hex" end, @@ -639,6 +650,7 @@ defs.in_zero = setmetatable({ defs.in_decimal = setmetatable(selfify({ ['.'] = false, + numtype = TK_FLT }, "in_digit"), {__index=defs.in_integer}) defs.in_expnum = setmetatable(selfify({ @@ -654,6 +666,7 @@ defs.in_subexp = setmetatable({ }, {__index=defs.base}) defs.in_exp = setmetatable({ + in_subexp = defs.in_subexp, ["+"] = "sign", ["-"] = "sign", sign = function(state, token) @@ -662,18 +675,19 @@ defs.in_exp = setmetatable({ end, }, {__index=defs.in_subexp}) -defs.in_hex = setmetatable({ +defs.in_hex = setmetatable(selfify({ in_decimal = "in_hex_fraction", hexdigit = 'digit', ['e'] = 'hexdigit', ['E'] = 'hexdigit', ['p'] = 'exp', ['P'] = 'exp', -}, {__index=defs.in_integer}) +}, "in_digit"), {__index=defs.in_integer}) -defs.in_hex_fraction = setmetatable({ +defs.in_hex_fraction = setmetatable(selfify({ ['.'] = false, -}, {__index=defs.in_hex}) + numtype = TK_FLT +}, "in_digit"), {__index=defs.in_hex}) function defs.simpletoken(state, token) state[#state+1] = token diff --git a/test.lua b/test.lua index 077f139..5a1bd4d 100644 --- a/test.lua +++ b/test.lua @@ -405,6 +405,53 @@ do -- long comments end end -- long comments +do -- numbers + local luatokens = require "luatokens" + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, [[ + 3 345 0xff 0xBEBADA + 3.0 3.1416 314.16e-2 0.31416E1 34e1 + 0x0.1E 0xA23p-4 0X1.921FB54442D18P+1]]) + local case = case() + if not state then + print(case, "---- IN TOKENS ----") + print(case, err, etoken) + for i,v in pairs(estate) do + print(case, i, v) + end + print(case, "---- OUT TOKENS ----") + else + -- integers + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 3) + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 345) + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 0xff) + assert(table.remove(state, 1) == luatokens.tokens.TK_INT) + assert(table.remove(state, 1) == 0xBEBADA) + -- floats + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 3.0) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 3.1416) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 314.16e-2) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 0.31416E1) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 34e1) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 0x0.1E) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 0xA23p-4) + assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) + assert(table.remove(state, 1) == 0x1.921FB54442D18P+1) + assert(table.remove(state, 1) == nil) + assert(state.line == 3) + end +end -- numbers + do -- FUCK local luatokens = require "luatokens" local luatokens_file = io.open("./luatokens.lua", "r") |