diff options
Diffstat (limited to 'luatokens.lua')
| -rw-r--r-- | luatokens.lua | 30 |
1 files changed, 15 insertions, 15 deletions
diff --git a/luatokens.lua b/luatokens.lua index 81cbc11..2ac2cc3 100644 --- a/luatokens.lua +++ b/luatokens.lua @@ -3,6 +3,9 @@ -- we need some stuff from here local parser = require "parser" local selfify = parser.selfify +local EOF = parser.EOF +local COLLECT = parser.COLLECT +local collect_fallback = parser.collect_fallback -- "dummies" local TK_STRING = {} @@ -106,25 +109,25 @@ do local tstring = selfify({}) ["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}), ["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}), [1] = linecount, - [2] = print }, {__index = tokens.base}) tokens.string.escapes = tsescapes tsescapes.string = tokens.string function tsescapes.insertraw(state, token) - state[#state+1] = token + collect_fallback(state, token) return "string" end do local map = { ["a"] = "\a", ["b"] = "\b", ["f"] = "\f", ["n"] = "\n", ["r"] = "\r", ["t"] = "\t", ["v"] = "\v" } function tsescapes.insertmap(state, token) - state[#state+1] = map[token] + collect_fallback(state, map[token]) return "string" end end function tsescapes.digit(state, token) + print(state, token) local digit = string.find("1234567890", token, 1, true) local num = state.in_digit if digit then @@ -138,21 +141,21 @@ do local tstring = selfify({}) if num > 255 then return nil end - state[#state+1] = string.char(num) + collect_fallback(state, string.char(num)) state.in_digit = nil state.c = nil return "string" end tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, digitc = "self", string = tstring}), {__index=tstring}) - tsescapes.hex = setmetatable(selfify({string = tokens.string}), {__index=tokens.base}) + tsescapes.hex = setmetatable(selfify({string = tokens.string, digit = "hexdigit"}), {__index=tokens.base}) function tsescapes.hex.hexdigit(state, token) local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true) assert(digit, "this should never be called for non-hex-digits") local num = state.in_hex if num then num = num * 16 + digit % 16 - state[#state+1] = string.char(num) + collect_fallback(state, string.char(num)) state.in_hex = nil return "string" else @@ -165,7 +168,7 @@ do local tstring = selfify({}) string = tokens.string, whitespace = "self", [""] = "string", - [1] = parser.insert_fallback, + [1] = collect_fallback, [2] = linecount, }) local tbase = tokens.base @@ -185,17 +188,13 @@ do local tstring = selfify({}) tstring[""] = "self" - tstring[1] = parser.insert_fallback + tstring[1] = collect_fallback function tstring.close(state, token) if state.in_string == token then - local i = state.string_start state.in_string = nil - state.string_start = nil - state[i+1] = table.concat(state, '', i+1) - for j=i+2, #state do - state[j]=nil - end + state[#state+1] = table.concat(state[COLLECT]) + state[COLLECT] = nil return "tokens" else state[#state+1] = token @@ -206,14 +205,15 @@ end tokens["'"] = "string_open" tokens['"'] = "string_open" +tokens[1] = linecount setmetatable(tokens, {__index=whitespace}) function tokens.string_open(state, token) if not state.in_string then state[#state+1] = TK_STRING + state[COLLECT] = {} state.in_string = token - state.string_start = #state return "string" end assert("this shouldn't happen") |