Fix number parsing (not compatible with 5.1)

1 files changed, 19 insertions, 5 deletions

diff --git a/luatokens.lua b/luatokens.lua
index 5217ab5..ecc3b6d 100644
--- a/luatokens.lua
+++ b/luatokens.lua
@@ -616,6 +616,7 @@ defs.in_integer = setmetatable(selfify({
     alpha = false,
     ['e'] = "exp",
     ['E'] = "exp",
+    [parser.EOZ] = "self", -- defs
     exp = function(state, token)
         collect_fallback(state, token)
         return "in_exp"
@@ -628,10 +629,20 @@ defs.in_integer = setmetatable(selfify({
         collect_fallback(state, token)
         return "in_digit"
     end,
+    [-1] = function(state, token, rule)
+        -- TODO figure out best order for these checks
+        if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end
+        state[#state+1] = state[STATE].numtype
+        state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option
+        state[COLLECT] = nil
+    end,
+    numtype = TK_INT
 }, "in_digit"), {__index=defs})
 
 defs.in_zero = setmetatable({
-    ['x'] = function(state, token)
+    ['x'] = "into_hex",
+    ['X'] = "into_hex",
+    into_hex = function(state, token)
         collect_fallback(state, token)
         return "in_hex"
     end,
@@ -639,6 +650,7 @@ defs.in_zero = setmetatable({
 
 defs.in_decimal = setmetatable(selfify({
     ['.'] = false,
+    numtype = TK_FLT
 }, "in_digit"), {__index=defs.in_integer})
 
 defs.in_expnum = setmetatable(selfify({
@@ -654,6 +666,7 @@ defs.in_subexp = setmetatable({
 }, {__index=defs.base})
 
 defs.in_exp = setmetatable({
+    in_subexp = defs.in_subexp,
     ["+"] = "sign",
     ["-"] = "sign",
     sign = function(state, token)
@@ -662,18 +675,19 @@ defs.in_exp = setmetatable({
     end,
 }, {__index=defs.in_subexp})
 
-defs.in_hex = setmetatable({
+defs.in_hex = setmetatable(selfify({
     in_decimal = "in_hex_fraction",
     hexdigit = 'digit',
     ['e'] = 'hexdigit',
     ['E'] = 'hexdigit',
     ['p'] = 'exp',
     ['P'] = 'exp',
-}, {__index=defs.in_integer})
+}, "in_digit"), {__index=defs.in_integer})
 
-defs.in_hex_fraction = setmetatable({
+defs.in_hex_fraction = setmetatable(selfify({
     ['.'] = false,
-}, {__index=defs.in_hex})
+    numtype = TK_FLT
+}, "in_digit"), {__index=defs.in_hex})
 
 function defs.simpletoken(state, token)
     state[#state+1] = token