Tokenize numbers as strings

"Fixes" Lua 5.2 support, but makes certain number formats non-portable,
for example hex floats will not work before Lua 5.3.

2 files changed, 3 insertions, 23 deletions

diff --git a/src/cratera/compiler.lua b/src/cratera/compiler.lua
index 5e9b0b5..ab4e2bf 100644
--- a/src/cratera/compiler.lua
+++ b/src/cratera/compiler.lua
@@ -73,30 +73,10 @@ finish[parser.EOZ] = function(state, token)
     local tk = table.remove(results)
     if tk == TK.FLT then
         local token = table.remove(results)
-        local extra, num, den = 1, token, 1
-        assert(token == token and token >= 0, "NYI") -- the tokenizer should never output NaNs or negative values
-        if token == math.huge then -- the tokenizer *can* output math.huge tho
-            num, den = 1, 0
-        else
-            while num ~= math.floor(num) do
-                num = num * 2 -- always safe (I think)
-                local oldden = den
-                den = den * 2
-                if den == math.huge then -- subnormals or something?
-                    extra = oldden
-                    den = 2
-                end
-            end
-        end
-        table.insert(state, string.format('((%d/%d)/%d)', num, den, extra))
+        table.insert(state, token)
     elseif tk == TK.INT then
         local v = table.remove(results)
-        if v == math.mininteger then
-            -- corner case ( https://github.com/lua/lua/commit/707b0ba6e2dbfd58cf1167dae0e17975904b18aa )
-            table.insert(state, string.format('0x%x', v))
-        else
-            table.insert(state, string.format('(%d)', v)) -- may be negative (overflow)
-        end
+        table.insert(state, v)
     elseif tk == TK.STRING then
         -- lua tends to use a backslash and a newline but we already do newline processing,
         -- so we need to replace the escaped newline ("\\\n") with a newline escape ("\\n").
diff --git a/src/cratera/luatokens.lua b/src/cratera/luatokens.lua
index 632cf27..6958171 100644
--- a/src/cratera/luatokens.lua
+++ b/src/cratera/luatokens.lua
@@ -641,7 +641,7 @@ defs.in_integer = setmetatable(selfify({
         -- TODO figure out best order for these checks
         if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end
         state[#state+1] = state[STATE].numtype
-        state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option
+        state[#state+1] = table.concat(state[COLLECT]) -- TODO maybe not the best option
         state[COLLECT] = nil
     end,
     numtype = TK_INT