path: root/luatokens.lua

-- Lua defs

-- we need some stuff from here
local parser = require "parser"
local selfify = parser.selfify
local EOF = parser.EOF
local COLLECT = parser.COLLECT
local collect_fallback = parser.collect_fallback

-- "dummies"
-- see http://www.lua.org/source/5.3/llex.h.html#RESERVED
-- keywords
local TK_AND, TK_BREAK,
    TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
    TK_GOTO, TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
    TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
    -- operators
    TK_IDIV, TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE,
    TK_SHL, TK_SHR,
    -- misc
    TK_DBCOLON, TK_EOS,
    TK_FLT, TK_INT, TK_NAME, TK_STRING =
    {}, {},
    {}, {}, {}, {}, {}, {}, {},
    {}, {}, {}, {}, {}, {}, {}, {},
    {}, {}, {}, {}, {},
    {}, {}, {}, {}, {}, {}, {},
    {}, {},
    {}, {},
    {}, {}, {}, {}

local keywords = {
    ["and"] = TK_AND,
    ["break"] = TK_BREAK,
    ["do"] = TK_DO,
    ["else"] = TK_ELSE,
    ["elseif"] = TK_ELSEIF,
    ["end"] = TK_END,
    ["false"] = TK_FALSE,
    ["for"] = TK_FOR,
    ["function"] = TK_FUNCTION,
    ["goto"] = TK_GOTO,
    ["if"] = TK_IF,
    ["in"] = TK_IN,
    ["local"] = TK_LOCAL,
    ["nil"] = TK_NIL,
    ["not"] = TK_NOT,
    ["or"] = TK_OR,
    ["repeat"] = TK_REPEAT,
    ["return"] = TK_RETURN,
    ["then"] = TK_THEN,
    ["true"] = TK_TRUE,
    ["until"] = TK_UNTIL,
    ["while"] = TK_WHILE,
}

local defs = selfify({})

defs.base = {
    [" "] = "whitespace",
    ["\n"] = "newline",
    ["\r"] = "newline",
    ["\v"] = "whitespace",
    ["\t"] = "whitespace",
    ["\f"] = "whitespace",
    ["0"] = "digit",
    ["1"] = "digit",
    ["2"] = "digit",
    ["3"] = "digit",
    ["4"] = "digit",
    ["5"] = "digit",
    ["6"] = "digit",
    ["7"] = "digit",
    ["8"] = "digit",
    ["9"] = "digit",
    ["a"] = "hexdigit",
    ["b"] = "hexdigit",
    ["c"] = "hexdigit",
    ["d"] = "hexdigit",
    ["e"] = "hexdigit",
    ["f"] = "hexdigit",
    ["A"] = "hexdigit",
    ["B"] = "hexdigit",
    ["C"] = "hexdigit",
    ["D"] = "hexdigit",
    ["E"] = "hexdigit",
    ["F"] = "hexdigit",
    ["g"] = "alpha",
    ["h"] = "alpha",
    ["i"] = "alpha",
    ["j"] = "alpha",
    ["k"] = "alpha",
    ["l"] = "alpha",
    ["m"] = "alpha",
    ["n"] = "alpha",
    ["o"] = "alpha",
    ["p"] = "alpha",
    ["q"] = "alpha",
    ["r"] = "alpha",
    ["s"] = "alpha",
    ["t"] = "alpha",
    ["u"] = "alpha",
    ["v"] = "alpha",
    ["w"] = "alpha",
    ["x"] = "alpha",
    ["y"] = "alpha",
    ["z"] = "alpha",
    ["G"] = "alpha",
    ["H"] = "alpha",
    ["I"] = "alpha",
    ["J"] = "alpha",
    ["K"] = "alpha",
    ["L"] = "alpha",
    ["M"] = "alpha",
    ["N"] = "alpha",
    ["O"] = "alpha",
    ["P"] = "alpha",
    ["Q"] = "alpha",
    ["R"] = "alpha",
    ["S"] = "alpha",
    ["T"] = "alpha",
    ["U"] = "alpha",
    ["V"] = "alpha",
    ["W"] = "alpha",
    ["X"] = "alpha",
    ["Y"] = "alpha",
    ["Z"] = "alpha",
}

local function countline(state, token, rule)
    state.line = (state.line or 1) + 1
end

local function mknewline(t, hookn, fallback)
    fallback = fallback or t
    t["\n"] = setmetatable({[hookn] = countline, ["\r"] = setmetatable({}, {__index=fallback})}, {__index=fallback})
    t["\r"] = setmetatable({[hookn] = countline, ["\n"] = setmetatable({}, {__index=fallback})}, {__index=fallback})
    return t
end

do local tstring = selfify({})
    defs.string = tstring
    tstring.defs = defs
    do local tsescapes = setmetatable(mknewline({
            ["'"] = "insertraw",
            ['"'] = "insertraw",
            ['\\'] = "insertraw",
            ["a"] = "insertmap",
            ["b"] = "insertmap",
            ["f"] = "insertmap",
            ["n"] = "insertmap",
            ["r"] = "insertmap",
            ["t"] = "insertmap",
            ["v"] = "insertmap",
            ["z"] = "skipwhitespace",
            ["u"] = "unicode",
            ["x"] = "hex",
            --["\n"] = setmetatable({[1] = countline, ["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
            --["\r"] = setmetatable({[1] = countline, ["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
            [1] = function(state, token, rule) if token == "\r" or token == "\n" then collect_fallback(state, "\n") end end,
        }, 1, tstring), {__index = defs.base})
        defs.string.escapes = tsescapes
        tsescapes.string = defs.string

        function tsescapes.insertraw(state, token)
            collect_fallback(state, token)
            return "string"
        end

        do
            local map = { ["a"] = "\a", ["b"] = "\b", ["f"] = "\f", ["n"] = "\n", ["r"] = "\r", ["t"] = "\t", ["v"] = "\v" }
            function tsescapes.insertmap(state, token)
                collect_fallback(state, map[token])
                return "string"
            end
        end

        function tsescapes.digit(state, token)
            local digit = string.find("1234567890", token, 1, true)
            local num = state.in_digit
            if digit then
                num = (num or 0) * 10 + digit % 10
                state.c = (state.c or 0) + 1
                if state.c < 3 then
                    state.in_digit = num
                    return "digitc"
                end
            end
            if num > 255 then
                return nil
            end
            collect_fallback(state, string.char(num))
            state.in_digit = nil
            state.c = nil
            if not digit then
                collect_fallback(state, token)
            end
            return "string"
        end
        tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring})
        tsescapes.digitc[1]=function(state, token, rule)
            if rule == nil then
                collect_fallback(state, string.char(state.in_digit))
                state.in_digit = nil
                state.c = nil
            end
        end

        tsescapes.hex = setmetatable(selfify({string = defs.string, digit = "hexdigit"}), {__index=defs.base})
        function tsescapes.hex.hexdigit(state, token)
            local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true)
            assert(digit, "this should never be called for non-hex-digits")
            local num = state.in_hex
            if num then
                num = num * 16 + digit % 16
                collect_fallback(state, string.char(num))
                state.in_hex = nil
                return "string"
            else
                state.in_hex = digit % 16
                return "self"
            end
        end

        do local tseunicode = {}
            tseunicode["{"] = "hex"
            do local tseuhex = setmetatable(selfify({digit = "hexdigit", string=tstring}), {__index=defs.base})
                tseunicode.hex = tseuhex
                function tseuhex.hexdigit(state, token)
                    local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true)
                    assert(digit, "this should never be called for non-hex-digits")
                    state.in_hex = (state.in_hex or 0) * 16 + digit % 16
                    if state.in_hex <= 2147483647 then
                        return "self"
                    end
                end
                tseuhex["}"] = function(state, token)
                    local num = state.in_hex
                    state.in_hex = nil
                    if num < 128 then
                        collect_fallback(state, string.char(num))
                        return "string"
                    end
                    local bytes = ""
                    while num > 63 do
                        local v = num % 64
                        bytes = string.char(128 + v) .. bytes -- yeah ik, not the most efficient
                        num = (num - v) / 64
                    end
                    if num >= 2^6/(2^#bytes) then
                        local v = num % 64
                        bytes = string.char(128 + v) .. bytes
                        num = (num - v) / 64
                    end
                    do
                        local v = 0
                        for i=1,#bytes do
                            v = v + 128 / 2^i
                        end
                        v = v + num
                        assert(v < 126)
                        bytes = string.char(128 + v) .. bytes
                    end
                    collect_fallback(state, bytes)
                    return "string"
                end
            end
            tsescapes.unicode = tseunicode
        end

        do local tseskipwhitespace = selfify(mknewline({
                string = defs.string,
                whitespace = "self",
                [""] = "string",
                [1] = collect_fallback,
            }, 2))
            --tseskipwhitespace["\n"] = setmetatable({[2] = countline, ["\r"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
            --tseskipwhitespace["\r"] = setmetatable({[2] = countline, ["\n"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
            local tbase = defs.base
            local tbasemap = {whitespace = "whitespace"}
            setmetatable(tseskipwhitespace, {__index = function(t, k) return tbasemap[tbase[k]] or tstring[k] end})
            tsescapes.skipwhitespace =  tseskipwhitespace
        end
    end

    tstring['\\'] = "escapes"

    tstring['"'] = "close"
    tstring["'"] = "close"

    tstring['\n'] = false
    tstring['\r'] = false

    tstring[""] = "self"

    tstring[1] = collect_fallback

    function tstring.close(state, token)
        if state.in_string == token then
            state.in_string = nil
            state[#state+1] = table.concat(state[COLLECT])
            state[COLLECT] = nil
            return "defs"
        else
            collect_fallback(state, token)
            return "self"
        end
    end
end

do local tlongstring = {}
    defs.longstring = tlongstring
    do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
        tllongstring_proper[1] = collect_fallback

        do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
            tllongstring_proper.maybe_end = tllmaybe_end
            tllmaybe_end["="] = function(state, token)
                state.longstring_close = state.longstring_close + 1
                return "maybe_end"
            end
            tllmaybe_end["]"] = function(state, token)
                if state.longstring_close == state.longstring_count then
                    state.longstring_close = nil
                    state.longstring_count = nil
                    local pos = #state
                    state[pos+1] = TK_STRING
                    state[pos+2] = table.concat(state[COLLECT])
                    state[COLLECT] = nil
                    return "defs"
                else
                    collect_fallback(state, "]")
                    collect_fallback(state, ("="):rep(state.longstring_close))
                    state.longstring_close = 0
                    return "maybe_end"
                end
            end
            tllmaybe_end[-1] = function(state, token, rule)
                if not rule then
                    collect_fallback(state, "]")
                    collect_fallback(state, ("="):rep(state.longstring_close))
                    state.longstring_close = nil
                end
            end
        end

        tlongstring.longstring_proper = tllongstring_proper
        mknewline(tlongstring, 1, tllongstring_proper)
        setmetatable(tlongstring, {__index=tllongstring_proper})
    end
end

defs["'"] = "string_open"
defs['"'] = "string_open"
defs["["] = "maybe_longstring"
defs.maybe_longstring = setmetatable({
    defs = defs,
    ['['] = "longstring_open",
    ['='] = "longstring_open",
    longstring_count = selfify({
        ["="] = function(state, token)
            state.longstring_count = state.longstring_count + 1
            return "self"
        end,
        ["["] = function(state, token)
            state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
            return "longstring"
        end,
        longstring = defs.longstring
    }),
    longstring_open = function(state, token)
        if token == "=" then
            state.longstring_count = state.longstring_count or 0 + 1
            return "longstring_count"
        elseif token == "[" then
            state.longstring_count = 0
            state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
            return "longstring"
        end
    end,
    [-1] = function(state, token, rule)
        if rule ~= "longstring_open" then
            state[#state+1] = "["
        end
    end
}, {__index=defs})

-- these are needed for proper line counts
--defs["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=defs})}, {__index=defs})
--defs["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=defs})}, {__index=defs})
mknewline(defs, 1)

defs.whitespace = "self"
defs.hexdigit = "alpha"
defs["_"] = "alpha"
defs.in_alpha = setmetatable(selfify({digit = "in_alpha", hexdigit = "in_alpha", alpha = "in_alpha", _ = "in_alpha", [parser.EOZ] = "self"}, "in_alpha"), {__index=defs})
function defs.alpha(state, token)
    state[COLLECT] = {coalesce=15} -- TODO tweak this for CPU/memory tradeoff?
    collect_fallback(state, token)
    return "in_alpha"
end
defs.in_alpha[-1] = function(state, token, rule)
    if rule == "alpha" or rule == "digit" or rule == "hexdigit" or token == "_" then
        collect_fallback(state, token)
    else
        local key = table.concat(state[COLLECT])
        state[COLLECT] = nil
        local keyword = keywords[key]
        if keyword then
            state[#state+1] = keyword
        else
            local pos = #state
            state[pos+1] = TK_NAME
            state[pos+2] = key
        end
    end
end

setmetatable(defs, {__index=defs.base})

function defs.string_open(state, token)
    if not state.in_string then
        state[#state+1] = TK_STRING
        state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
        state.in_string = token
        return "string"
    end
    assert("this shouldn't happen")
end

return {
    defs = defs,
    tokens = {
        TK_AND = TK_AND, TK_BREAK = TK_BREAK,
        TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION,
        TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT,
        TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE,
        TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE,
        TK_SHL = TK_SHL, TK_SHR = TK_SHR,
        TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS,
        TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING
    },
}