Cleaning up

1 files changed, 769 insertions, 0 deletions

diff --git a/cratera/luatokens.lua b/cratera/luatokens.lua
new file mode 100644
index 0000000..90d0c61
--- /dev/null
+++ b/cratera/luatokens.lua
@@ -0,0 +1,769 @@
+--[[
+    This file is part of Cratera Compiler
+    Copyright (C) 2019  Soni L.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+--]]
+
+--[[
+    This software is based on Lua 5.1 and Lua 5.3
+
+    Lua 5.1 license:
+
+/******************************************************************************
+* Copyright (C) 1994-2012 Lua.org, PUC-Rio.  All rights reserved.
+*
+* Permission is hereby granted, free of charge, to any person obtaining
+* a copy of this software and associated documentation files (the
+* "Software"), to deal in the Software without restriction, including
+* without limitation the rights to use, copy, modify, merge, publish,
+* distribute, sublicense, and/or sell copies of the Software, and to
+* permit persons to whom the Software is furnished to do so, subject to
+* the following conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+    Lua 5.3 license:
+
+/******************************************************************************
+* Copyright (C) 1994-2018 Lua.org, PUC-Rio.
+*
+* Permission is hereby granted, free of charge, to any person obtaining
+* a copy of this software and associated documentation files (the
+* "Software"), to deal in the Software without restriction, including
+* without limitation the rights to use, copy, modify, merge, publish,
+* distribute, sublicense, and/or sell copies of the Software, and to
+* permit persons to whom the Software is furnished to do so, subject to
+* the following conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+--]]
+
+-- we need some stuff from here
+local parser = require "cratera.parser"
+local selfify = parser.selfify
+local EOF = parser.EOF
+local COLLECT = parser.COLLECT
+local collect_fallback = parser.collect_fallback
+
+-- "dummies"
+-- see http://www.lua.org/source/5.3/llex.h.html#RESERVED
+-- keywords
+local TK_AND, TK_BREAK,
+    TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
+    TK_GOTO, TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
+    TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
+    -- operators
+    TK_IDIV, TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE,
+    TK_SHL, TK_SHR,
+    -- misc
+    TK_DBCOLON, TK_EOS,
+    -- values/constants
+    TK_FLT, TK_INT, TK_NAME, TK_STRING =
+    {}, {},
+    {}, {}, {}, {}, {}, {}, {},
+    {}, {}, {}, {}, {}, {}, {}, {},
+    {}, {}, {}, {}, {},
+    {}, {}, {}, {}, {}, {}, {},
+    {}, {},
+    {}, {},
+    {}, {}, {}, {}
+
+local keywords = {
+    ["and"] = TK_AND,
+    ["break"] = TK_BREAK,
+    ["do"] = TK_DO,
+    ["else"] = TK_ELSE,
+    ["elseif"] = TK_ELSEIF,
+    ["end"] = TK_END,
+    ["false"] = TK_FALSE,
+    ["for"] = TK_FOR,
+    ["function"] = TK_FUNCTION,
+    ["goto"] = TK_GOTO,
+    ["if"] = TK_IF,
+    ["in"] = TK_IN,
+    ["local"] = TK_LOCAL,
+    ["nil"] = TK_NIL,
+    ["not"] = TK_NOT,
+    ["or"] = TK_OR,
+    ["repeat"] = TK_REPEAT,
+    ["return"] = TK_RETURN,
+    ["then"] = TK_THEN,
+    ["true"] = TK_TRUE,
+    ["until"] = TK_UNTIL,
+    ["while"] = TK_WHILE,
+}
+
+local reverse_keywords = {}
+for k,v in pairs(keywords) do
+    reverse_keywords[v] = k
+end
+
+local defs = selfify({})
+
+defs.base = {
+    [" "] = "whitespace",
+    ["\n"] = "newline",
+    ["\r"] = "newline",
+    ["\v"] = "whitespace",
+    ["\t"] = "whitespace",
+    ["\f"] = "whitespace",
+    ["0"] = "digit",
+    ["1"] = "digit",
+    ["2"] = "digit",
+    ["3"] = "digit",
+    ["4"] = "digit",
+    ["5"] = "digit",
+    ["6"] = "digit",
+    ["7"] = "digit",
+    ["8"] = "digit",
+    ["9"] = "digit",
+    ["a"] = "hexdigit",
+    ["b"] = "hexdigit",
+    ["c"] = "hexdigit",
+    ["d"] = "hexdigit",
+    ["e"] = "hexdigit",
+    ["f"] = "hexdigit",
+    ["A"] = "hexdigit",
+    ["B"] = "hexdigit",
+    ["C"] = "hexdigit",
+    ["D"] = "hexdigit",
+    ["E"] = "hexdigit",
+    ["F"] = "hexdigit",
+    ["g"] = "alpha",
+    ["h"] = "alpha",
+    ["i"] = "alpha",
+    ["j"] = "alpha",
+    ["k"] = "alpha",
+    ["l"] = "alpha",
+    ["m"] = "alpha",
+    ["n"] = "alpha",
+    ["o"] = "alpha",
+    ["p"] = "alpha",
+    ["q"] = "alpha",
+    ["r"] = "alpha",
+    ["s"] = "alpha",
+    ["t"] = "alpha",
+    ["u"] = "alpha",
+    ["v"] = "alpha",
+    ["w"] = "alpha",
+    ["x"] = "alpha",
+    ["y"] = "alpha",
+    ["z"] = "alpha",
+    ["G"] = "alpha",
+    ["H"] = "alpha",
+    ["I"] = "alpha",
+    ["J"] = "alpha",
+    ["K"] = "alpha",
+    ["L"] = "alpha",
+    ["M"] = "alpha",
+    ["N"] = "alpha",
+    ["O"] = "alpha",
+    ["P"] = "alpha",
+    ["Q"] = "alpha",
+    ["R"] = "alpha",
+    ["S"] = "alpha",
+    ["T"] = "alpha",
+    ["U"] = "alpha",
+    ["V"] = "alpha",
+    ["W"] = "alpha",
+    ["X"] = "alpha",
+    ["Y"] = "alpha",
+    ["Z"] = "alpha",
+}
+
+local function countline(state, token, rule)
+    state.line = (state.line or 1) + 1
+end
+
+local function mknewline(t, hookn, fallback)
+    fallback = fallback or t
+    t["\n"] = setmetatable({[hookn] = countline, ["\r"] = setmetatable({}, {__index=fallback})}, {__index=fallback})
+    t["\r"] = setmetatable({[hookn] = countline, ["\n"] = setmetatable({}, {__index=fallback})}, {__index=fallback})
+    return t
+end
+
+do local tstring = selfify({})
+    defs.string = tstring
+    tstring.defs = defs
+    do local tsescapes = setmetatable(mknewline({
+            ["'"] = "insertraw",
+            ['"'] = "insertraw",
+            ['\\'] = "insertraw",
+            ["a"] = "insertmap",
+            ["b"] = "insertmap",
+            ["f"] = "insertmap",
+            ["n"] = "insertmap",
+            ["r"] = "insertmap",
+            ["t"] = "insertmap",
+            ["v"] = "insertmap",
+            ["z"] = "skipwhitespace",
+            ["u"] = "unicode",
+            ["x"] = "hex",
+            --["\n"] = setmetatable({[1] = countline, ["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
+            --["\r"] = setmetatable({[1] = countline, ["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
+            [1] = function(state, token, rule) if token == "\r" or token == "\n" then collect_fallback(state, "\n") end end,
+        }, 1, tstring), {__index = defs.base})
+        defs.string.escapes = tsescapes
+        tsescapes.string = defs.string
+
+        function tsescapes.insertraw(state, token)
+            collect_fallback(state, token)
+            return "string"
+        end
+
+        do
+            local map = { ["a"] = "\a", ["b"] = "\b", ["f"] = "\f", ["n"] = "\n", ["r"] = "\r", ["t"] = "\t", ["v"] = "\v" }
+            function tsescapes.insertmap(state, token)
+                collect_fallback(state, map[token])
+                return "string"
+            end
+        end
+
+        function tsescapes.digit(state, token)
+            local digit = string.find("1234567890", token, 1, true)
+            local num = state.in_digit
+            if digit then
+                num = (num or 0) * 10 + digit % 10
+                state.c = (state.c or 0) + 1
+                if state.c < 3 then
+                    state.in_digit = num
+                    return "digitc"
+                end
+            end
+            if num > 255 then
+                return nil
+            end
+            collect_fallback(state, string.char(num))
+            state.in_digit = nil
+            state.c = nil
+            if not digit then
+                collect_fallback(state, token)
+            end
+            return "string"
+        end
+        tsescapes.digitc = setmetatable(selfify({[parser.FALLBACK] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring})
+        tsescapes.digitc[1]=function(state, token, rule)
+            if rule == nil then
+                collect_fallback(state, string.char(state.in_digit))
+                state.in_digit = nil
+                state.c = nil
+            end
+        end
+
+        tsescapes.hex = setmetatable(selfify({string = defs.string, digit = "hexdigit"}), {__index=defs.base})
+        function tsescapes.hex.hexdigit(state, token)
+            local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true)
+            assert(digit, "this should never be called for non-hex-digits")
+            local num = state.in_hex
+            if num then
+                num = num * 16 + digit % 16
+                collect_fallback(state, string.char(num))
+                state.in_hex = nil
+                return "string"
+            else
+                state.in_hex = digit % 16
+                return "self"
+            end
+        end
+
+        do local tseunicode = {}
+            tseunicode["{"] = "hex"
+            do local tseuhex = setmetatable(selfify({digit = "hexdigit", string=tstring}), {__index=defs.base})
+                tseunicode.hex = tseuhex
+                function tseuhex.hexdigit(state, token)
+                    local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true)
+                    assert(digit, "this should never be called for non-hex-digits")
+                    state.in_hex = (state.in_hex or 0) * 16 + digit % 16
+                    if state.in_hex <= 2147483647 then
+                        return "self"
+                    end
+                end
+                tseuhex["}"] = function(state, token)
+                    local num = state.in_hex
+                    state.in_hex = nil
+                    if num < 128 then
+                        collect_fallback(state, string.char(num))
+                        return "string"
+                    end
+                    local bytes = ""
+                    while num > 63 do
+                        local v = num % 64
+                        bytes = string.char(128 + v) .. bytes -- yeah ik, not the most efficient
+                        num = (num - v) / 64
+                    end
+                    if num >= 2^6/(2^#bytes) then
+                        local v = num % 64
+                        bytes = string.char(128 + v) .. bytes
+                        num = (num - v) / 64
+                    end
+                    do
+                        local v = 0
+                        for i=1,#bytes do
+                            v = v + 128 / 2^i
+                        end
+                        v = v + num
+                        assert(v < 126)
+                        bytes = string.char(128 + v) .. bytes
+                    end
+                    collect_fallback(state, bytes)
+                    return "string"
+                end
+            end
+            tsescapes.unicode = tseunicode
+        end
+
+        do local tseskipwhitespace = selfify(mknewline({
+                string = defs.string,
+                whitespace = "self",
+                [parser.FALLBACK] = "string",
+                [1] = collect_fallback,
+            }, 2))
+            --tseskipwhitespace["\n"] = setmetatable({[2] = countline, ["\r"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
+            --tseskipwhitespace["\r"] = setmetatable({[2] = countline, ["\n"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
+            local tbase = defs.base
+            local tbasemap = {whitespace = "whitespace"}
+            setmetatable(tseskipwhitespace, {__index = function(t, k) return tbasemap[tbase[k]] or tstring[k] end})
+            tsescapes.skipwhitespace =  tseskipwhitespace
+        end
+    end
+
+    tstring['\\'] = "escapes"
+
+    tstring['"'] = "close"
+    tstring["'"] = "close"
+
+    tstring['\n'] = false
+    tstring['\r'] = false
+
+    tstring[parser.FALLBACK] = "self"
+
+    tstring[1] = collect_fallback
+
+    function tstring.close(state, token)
+        if state.in_string == token then
+            state.in_string = nil
+            state[#state+1] = table.concat(state[COLLECT])
+            state[COLLECT] = nil
+            return "defs"
+        else
+            collect_fallback(state, token)
+            return "self"
+        end
+    end
+end
+
+do local tlongstring = {}
+    defs.longstring = tlongstring
+    do local tllongstring_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
+        tllongstring_proper[1] = false -- placeholder for newline handling
+        tllongstring_proper[2] = collect_fallback
+
+        do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
+            tllongstring_proper.maybe_end = tllmaybe_end
+            tllmaybe_end.longstring_proper = tllongstring_proper
+            tllmaybe_end["="] = function(state, token)
+                state.longstring_close = state.longstring_close + 1
+                return "maybe_end"
+            end
+            tllmaybe_end["]"] = function(state, token)
+                if state.longstring_close == state.longstring_count then
+                    state.longstring_close = nil
+                    state.longstring_count = nil
+                    local pos = #state
+                    state[pos+1] = TK_STRING
+                    state[pos+2] = table.concat(state[COLLECT])
+                    state[COLLECT] = nil
+                    return "defs"
+                else
+                    collect_fallback(state, "]")
+                    collect_fallback(state, ("="):rep(state.longstring_close))
+                    state.longstring_close = 0
+                    return "maybe_end"
+                end
+            end
+            tllmaybe_end[parser.FALLBACK] = "longstring_proper"
+            tllmaybe_end[1] = collect_fallback
+            tllmaybe_end[-1] = function(state, token, rule)
+                if not rule then
+                    collect_fallback(state, "]")
+                    collect_fallback(state, ("="):rep(state.longstring_close))
+                    state.longstring_close = nil
+                end
+            end
+        end
+
+        tlongstring.longstring_proper = tllongstring_proper
+        mknewline(tlongstring, 1, tllongstring_proper)
+        setmetatable(tlongstring, {__index=tllongstring_proper})
+    end
+end
+
+defs["'"] = "string_open"
+defs['"'] = "string_open"
+defs["["] = "maybe_longstring"
+defs.maybe_longstring = setmetatable({
+    defs = defs,
+    ['['] = "longstring_open",
+    ['='] = "longstring_open",
+    longstring_count = selfify({
+        ["="] = function(state, token)
+            state.longstring_count = state.longstring_count + 1
+            return "self"
+        end,
+        ["["] = function(state, token)
+            state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
+            return "longstring"
+        end,
+        longstring = defs.longstring
+    }),
+    longstring_open = function(state, token)
+        if token == "=" then
+            state.longstring_count = state.longstring_count or 0 + 1
+            return "longstring_count"
+        elseif token == "[" then
+            state.longstring_count = 0
+            state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
+            return "longstring"
+        end
+    end,
+    [-1] = function(state, token, rule)
+        if rule ~= "longstring_open" then
+            state[#state+1] = "["
+        end
+    end
+}, {__index=defs})
+
+-- these are needed for proper line counts
+--defs["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=defs})}, {__index=defs})
+--defs["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=defs})}, {__index=defs})
+mknewline(defs, 1)
+
+-- thankfully comments are easy
+defs["-"] = "maybe_comment"
+do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
+    defs.maybe_comment = tmaybe_comment
+    tmaybe_comment[parser.EOZ] = "self" -- defs
+    tmaybe_comment[-1] = function(state, token, rule)
+        if rule ~= "comment" then
+            state[#state+1] = "-"
+        end
+    end
+    do local tmcomment = {comment_proper = selfify({})}
+        tmaybe_comment.comment = tmcomment
+        tmcomment[parser.FALLBACK] = "comment_proper"
+        tmcomment["["] = "maybe_longcomment"
+        mknewline(tmcomment, 1, defs)
+        mknewline(tmcomment.comment_proper, 1, defs)
+        tmcomment.comment_proper[parser.FALLBACK] = "self"
+        do local tllongcomment_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
+            tmcomment.longcomment = tllongcomment_proper
+            do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
+                tllongcomment_proper.maybe_end = tllmaybe_end
+                tllmaybe_end.longcomment_proper = tllongcomment_proper
+                tllmaybe_end["="] = function(state, token)
+                    state.longcomment_close = state.longcomment_close + 1
+                    return "maybe_end"
+                end
+                tllmaybe_end["]"] = function(state, token)
+                    if state.longcomment_close == state.longcomment_count then
+                        state.longcomment_close = nil
+                        state.longcomment_count = nil
+                        return "defs"
+                    else
+                        state.longcomment_close = 0
+                        return "maybe_end"
+                    end
+                end
+                tllmaybe_end[parser.FALLBACK] = "longcomment_proper"
+                tllmaybe_end[-1] = function(state, token, rule)
+                    if not rule then
+                        state.longcomment_close = nil
+                    end
+                end
+            end
+
+            mknewline(tllongcomment_proper, 1, tllongcomment_proper)
+        end
+
+        tmcomment.maybe_longcomment = setmetatable({
+            comment = tmcomment,
+            ['['] = "longcomment_open",
+            ['='] = "longcomment_open",
+            longcomment_count = setmetatable(selfify({
+                ["="] = function(state, token)
+                    state.longcomment_count = state.longcomment_count + 1
+                    return "longcomment_count"
+                end,
+                ["["] = "longcomment",
+                longcomment = tmcomment.longcomment,
+            }, "longcomment_count"), {__index=tmcomment}),
+            longcomment_open = function(state, token)
+                if token == "=" then
+                    state.longcomment_count = state.longcomment_count or 0 + 1
+                    return "longcomment_count"
+                elseif token == "[" then
+                    state.longcomment_count = 0
+                    return "longcomment"
+                end
+            end,
+        }, {__index=tmcomment})
+    end
+end
+
+local STATE = parser.STATE
+
+defs.multitokens = setmetatable({
+    [parser.EOZ] = "self",
+    [-1] = function(state, token, rule)
+        if not state[STATE].multitoken[token] then
+            state[#state+1] = state[STATE].first
+        end
+    end,
+    second = function(state, token)
+        state[#state+1] = state[STATE].multitoken[token]
+        return "self" -- actually goes into defs
+    end
+}, {
+    __index=defs,
+    __call=function(t, first, ...)
+        local function helper(t, second, result, ...)
+            if not second then return end
+            t[second] = "second"
+            t.multitoken[second] = result
+            return helper(t, ...)
+        end
+        defs[first] = setmetatable({
+            first = first,
+            multitoken = {}
+        }, {__index=t})
+        return helper(defs[first], ...)
+    end
+})
+
+defs.multitokens("=", "=", TK_EQ)
+defs.multitokens("/", "/", TK_IDIV)
+defs.multitokens("<", "<", TK_SHL, "=", TK_LE)
+defs.multitokens(">", ">", TK_SHR, "=", TK_GE)
+defs.multitokens("~", "=", TK_NE)
+defs.multitokens(":", ":", TK_DBCOLON)
+
+defs["."] = setmetatable({
+    [-1] = function(state, token, rule)
+        if token ~= "." then
+            if rule ~= "digit" then
+                state[#state+1] = "."
+            end
+        end
+    end,
+    digit = function(state, token, rule)
+        state[#state+1] = TK_FLT
+        state[COLLECT] = {".", coalesce=31}
+        return "in_decimal"
+    end,
+    ["."] = setmetatable({
+        [-1] = function(state, token, rule)
+            if token ~= "." then
+                state[#state+1] = TK_CONCAT
+            end
+        end,
+        ["."] = function(state, token)
+            state[#state+1] = TK_DOTS
+            return "self" -- actually goes into defs
+        end,
+    }, {__index=defs})
+}, {__index=defs})
+
+function defs.digit(state, token)
+    state[COLLECT] = {token, coalesce=31}
+    if token == "0" then
+        return "in_zero"
+    else
+        return "in_integer"
+    end
+end
+
+defs.in_integer = setmetatable(selfify({
+    hexdigit = "alpha",
+    alpha = false,
+    ['e'] = "exp",
+    ['E'] = "exp",
+    [parser.EOZ] = "self", -- defs
+    exp = function(state, token)
+        collect_fallback(state, token)
+        return "in_exp"
+    end,
+    ['.'] = function(state, token)
+        collect_fallback(state, token)
+        return "in_decimal"
+    end,
+    digit = function(state, token)
+        collect_fallback(state, token)
+        return "in_digit"
+    end,
+    [-1] = function(state, token, rule)
+        -- TODO figure out best order for these checks
+        if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end
+        state[#state+1] = state[STATE].numtype
+        state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option
+        state[COLLECT] = nil
+    end,
+    numtype = TK_INT
+}, "in_digit"), {__index=defs})
+
+defs.in_zero = setmetatable({
+    ['x'] = "into_hex",
+    ['X'] = "into_hex",
+    into_hex = function(state, token)
+        collect_fallback(state, token)
+        return "in_hex"
+    end,
+}, {__index=defs.in_integer})
+
+defs.in_decimal = setmetatable(selfify({
+    ['.'] = false,
+    numtype = TK_FLT
+}, "in_digit"), {__index=defs.in_integer})
+
+defs.in_expnum = setmetatable(selfify({
+    exp = false,
+}, "in_digit"), {__index=defs.in_decimal})
+
+defs.in_subexp = setmetatable({
+    in_expnum = defs.in_expnum,
+    digit = function(state, token)
+        collect_fallback(state, token)
+        return "in_expnum"
+    end,
+}, {__index=defs.base})
+
+defs.in_exp = setmetatable({
+    in_subexp = defs.in_subexp,
+    ["+"] = "sign",
+    ["-"] = "sign",
+    sign = function(state, token)
+        collect_fallback(state, token)
+        return "in_subexp"
+    end,
+}, {__index=defs.in_subexp})
+
+defs.in_hex = setmetatable(selfify({
+    in_decimal = "in_hex_fraction",
+    hexdigit = 'digit',
+    ['e'] = 'hexdigit',
+    ['E'] = 'hexdigit',
+    ['p'] = 'exp',
+    ['P'] = 'exp',
+}, "in_digit"), {__index=defs.in_integer})
+
+defs.in_hex_fraction = setmetatable(selfify({
+    ['.'] = false,
+    numtype = TK_FLT
+}, "in_digit"), {__index=defs.in_hex})
+
+function defs.simpletoken(state, token)
+    state[#state+1] = token
+    return "self"
+end
+
+for token in string.gmatch("+*%^#&|(){}];,", ".") do
+    defs[token] = "simpletoken"
+end
+
+defs.whitespace = "self"
+defs.hexdigit = "alpha"
+defs["_"] = "alpha"
+defs.in_alpha = setmetatable(selfify({digit = "in_alpha", hexdigit = "in_alpha", alpha = "in_alpha", _ = "in_alpha", [parser.EOZ] = "self"}, "in_alpha"), {__index=defs})
+function defs.alpha(state, token)
+    state[COLLECT] = {coalesce=15} -- TODO tweak this for CPU/memory tradeoff?
+    collect_fallback(state, token)
+    return "in_alpha"
+end
+defs.in_alpha[-1] = function(state, token, rule)
+    if rule == "alpha" or rule == "digit" or rule == "hexdigit" or token == "_" then
+        collect_fallback(state, token)
+    else
+        local key = table.concat(state[COLLECT])
+        state[COLLECT] = nil
+        local keyword = keywords[key]
+        if keyword then
+            state[#state+1] = keyword
+        else
+            local pos = #state
+            state[pos+1] = TK_NAME
+            state[pos+2] = key
+        end
+    end
+end
+
+setmetatable(defs, {__index=defs.base})
+
+function defs.string_open(state, token)
+    if not state.in_string then
+        state[#state+1] = TK_STRING
+        state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
+        state.in_string = token
+        return "string"
+    end
+    assert("this shouldn't happen")
+end
+
+local tokens = {
+    TK_AND = TK_AND, TK_BREAK = TK_BREAK,
+    TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION,
+    TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT,
+    TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE,
+    TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE,
+    TK_SHL = TK_SHL, TK_SHR = TK_SHR,
+    TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS,
+    TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING
+}
+local TK = {}
+for k,v in pairs(tokens) do
+    setmetatable(v, {__name=k, __tostring=function(self) return getmetatable(self).__name end})
+    TK[k:sub(4)] = v
+end
+
+return {
+    defs = defs,
+    tokens = tokens,
+    TK = TK,
+    reverse_keywords = reverse_keywords,
+    reverse_tokens = {
+        [TK_IDIV] = "//", [TK_CONCAT] = "..", [TK_DOTS] = "...", [TK_EQ] = "==", [TK_GE] = ">=", [TK_LE] = "<=", [TK_NE] = "~=",
+        [TK_SHL] = "<<", [TK_SHR] = ">>",
+        [TK_DBCOLON] = "::", [TK_EOS] = "<eof>",
+        [TK_FLT] = "<float>", [TK_INT] = "<integer>", [TK_NAME] = "<identifier>", [TK_STRING] = "<string>"
+    },
+}