diff options
Diffstat (limited to 'luatokens.lua')
| -rw-r--r-- | luatokens.lua | 225 |
1 files changed, 225 insertions, 0 deletions
diff --git a/luatokens.lua b/luatokens.lua new file mode 100644 index 0000000..81cbc11 --- /dev/null +++ b/luatokens.lua @@ -0,0 +1,225 @@ +-- Lua tokens + +-- we need some stuff from here +local parser = require "parser" +local selfify = parser.selfify + +-- "dummies" +local TK_STRING = {} + +local tokens = {} + +tokens.base = { + [" "] = "whitespace", + ["\n"] = "newline", + ["\r"] = "newline", + ["\v"] = "whitespace", + ["\t"] = "whitespace", + ["\f"] = "whitespace", + ["0"] = "digit", + ["1"] = "digit", + ["2"] = "digit", + ["3"] = "digit", + ["4"] = "digit", + ["5"] = "digit", + ["6"] = "digit", + ["7"] = "digit", + ["8"] = "digit", + ["9"] = "digit", + ["a"] = "hexdigit", + ["b"] = "hexdigit", + ["c"] = "hexdigit", + ["d"] = "hexdigit", + ["e"] = "hexdigit", + ["f"] = "hexdigit", + ["A"] = "hexdigit", + ["B"] = "hexdigit", + ["C"] = "hexdigit", + ["D"] = "hexdigit", + ["E"] = "hexdigit", + ["F"] = "hexdigit", + ["g"] = "alpha", + ["h"] = "alpha", + ["i"] = "alpha", + ["j"] = "alpha", + ["k"] = "alpha", + ["l"] = "alpha", + ["m"] = "alpha", + ["n"] = "alpha", + ["o"] = "alpha", + ["p"] = "alpha", + ["q"] = "alpha", + ["r"] = "alpha", + ["s"] = "alpha", + ["t"] = "alpha", + ["u"] = "alpha", + ["v"] = "alpha", + ["w"] = "alpha", + ["x"] = "alpha", + ["y"] = "alpha", + ["z"] = "alpha", + ["G"] = "alpha", + ["H"] = "alpha", + ["I"] = "alpha", + ["J"] = "alpha", + ["K"] = "alpha", + ["L"] = "alpha", + ["M"] = "alpha", + ["N"] = "alpha", + ["O"] = "alpha", + ["P"] = "alpha", + ["Q"] = "alpha", + ["R"] = "alpha", + ["S"] = "alpha", + ["T"] = "alpha", + ["U"] = "alpha", + ["V"] = "alpha", + ["W"] = "alpha", + ["X"] = "alpha", + ["Y"] = "alpha", + ["Z"] = "alpha", +} + +local function linecount(state, token, rule) + if token == "\n" or token == "\r" then + state.line = (state.line or 1) + 1 + end +end + +do local tstring = selfify({}) + tokens.string = tstring + tstring.tokens = tokens + do local tsescapes = setmetatable({ + ["'"] = "insertraw", + ['"'] = "insertraw", + ['\\'] = "insertraw", + ["a"] = "insertmap", + ["b"] = "insertmap", + ["f"] = "insertmap", + ["n"] = "insertmap", + ["r"] = "insertmap", + ["t"] = "insertmap", + ["v"] = "insertmap", + ["z"] = "skipwhitespace", + ["u"] = "unicode", + ["x"] = "hex", + ["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}), + ["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}), + [1] = linecount, + [2] = print + }, {__index = tokens.base}) + tokens.string.escapes = tsescapes + tsescapes.string = tokens.string + + function tsescapes.insertraw(state, token) + state[#state+1] = token + return "string" + end + + do + local map = { ["a"] = "\a", ["b"] = "\b", ["f"] = "\f", ["n"] = "\n", ["r"] = "\r", ["t"] = "\t", ["v"] = "\v" } + function tsescapes.insertmap(state, token) + state[#state+1] = map[token] + return "string" + end + end + + function tsescapes.digit(state, token) + local digit = string.find("1234567890", token, 1, true) + local num = state.in_digit + if digit then + num = (num or 0) * 10 + digit % 10 + state.c = (state.c or 0) + 1 + if state.c < 3 then + state.in_digit = num + return "digitc" + end + end + if num > 255 then + return nil + end + state[#state+1] = string.char(num) + state.in_digit = nil + state.c = nil + return "string" + end + tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, digitc = "self", string = tstring}), {__index=tstring}) + + tsescapes.hex = setmetatable(selfify({string = tokens.string}), {__index=tokens.base}) + function tsescapes.hex.hexdigit(state, token) + local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true) + assert(digit, "this should never be called for non-hex-digits") + local num = state.in_hex + if num then + num = num * 16 + digit % 16 + state[#state+1] = string.char(num) + state.in_hex = nil + return "string" + else + state.in_hex = digit % 16 + return "self" + end + end + + do local tseskipwhitespace = selfify({ + string = tokens.string, + whitespace = "self", + [""] = "string", + [1] = parser.insert_fallback, + [2] = linecount, + }) + local tbase = tokens.base + local tbasemap = {whitespace = "whitespace", newline = "whitespace"} + setmetatable(tseskipwhitespace, {__index = function(t, k) return tbasemap[tbase[k]] or tstring[k] end}) + tsescapes.skipwhitespace = tseskipwhitespace + end + end + + tstring['\\'] = "escapes" + + tstring['"'] = "close" + tstring["'"] = "close" + + tstring['\n'] = false + tstring['\r'] = false + + tstring[""] = "self" + + tstring[1] = parser.insert_fallback + + function tstring.close(state, token) + if state.in_string == token then + local i = state.string_start + state.in_string = nil + state.string_start = nil + state[i+1] = table.concat(state, '', i+1) + for j=i+2, #state do + state[j]=nil + end + return "tokens" + else + state[#state+1] = token + return "self" + end + end +end + +tokens["'"] = "string_open" +tokens['"'] = "string_open" + +setmetatable(tokens, {__index=whitespace}) + +function tokens.string_open(state, token) + if not state.in_string then + state[#state+1] = TK_STRING + state.in_string = token + state.string_start = #state + return "string" + end + assert("this shouldn't happen") +end + +return { + tokens = tokens, + TK_STRING = TK_STRING, +} |