diff options
| -rw-r--r-- | luatokens.lua | 16 | ||||
| -rw-r--r-- | parser.lua | 4 |
2 files changed, 15 insertions, 5 deletions
diff --git a/luatokens.lua b/luatokens.lua index 2ac2cc3..28944c1 100644 --- a/luatokens.lua +++ b/luatokens.lua @@ -109,6 +109,7 @@ do local tstring = selfify({}) ["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}), ["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}), [1] = linecount, + [2] = function(state, token, rule) if token == "\r" or token == "\n" then collect_fallback(state, "\n") end end, }, {__index = tokens.base}) tokens.string.escapes = tsescapes tsescapes.string = tokens.string @@ -127,7 +128,6 @@ do local tstring = selfify({}) end function tsescapes.digit(state, token) - print(state, token) local digit = string.find("1234567890", token, 1, true) local num = state.in_digit if digit then @@ -144,9 +144,19 @@ do local tstring = selfify({}) collect_fallback(state, string.char(num)) state.in_digit = nil state.c = nil + if not digit then + collect_fallback(state, token) + end return "string" end - tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, digitc = "self", string = tstring}), {__index=tstring}) + tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring}) + tsescapes.digitc[1]=function(state, token, rule) + if rule == nil then + collect_fallback(state, string.char(state.in_digit)) + state.in_digit = nil + state.c = nil + end + end tsescapes.hex = setmetatable(selfify({string = tokens.string, digit = "hexdigit"}), {__index=tokens.base}) function tsescapes.hex.hexdigit(state, token) @@ -197,7 +207,7 @@ do local tstring = selfify({}) state[COLLECT] = nil return "tokens" else - state[#state+1] = token + collect_fallback(state, token) return "self" end end diff --git a/parser.lua b/parser.lua index ff8378c..beb944e 100644 --- a/parser.lua +++ b/parser.lua @@ -134,8 +134,8 @@ return { stream = stream, parse = parse, -- common utility function - selfify = function(t) - t.self = t + selfify = function(t, id) + t[id or "self"] = t return t end, -- common hook |