diff options
Diffstat (limited to 'luatokens.lua')
| -rw-r--r-- | luatokens.lua | 45 |
1 files changed, 26 insertions, 19 deletions
diff --git a/luatokens.lua b/luatokens.lua index ecc3b6d..9ca19ac 100644 --- a/luatokens.lua +++ b/luatokens.lua @@ -272,7 +272,7 @@ do local tstring = selfify({}) end return "string" end - tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring}) + tsescapes.digitc = setmetatable(selfify({[parser.FALLBACK] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring}) tsescapes.digitc[1]=function(state, token, rule) if rule == nil then collect_fallback(state, string.char(state.in_digit)) @@ -346,7 +346,7 @@ do local tstring = selfify({}) do local tseskipwhitespace = selfify(mknewline({ string = defs.string, whitespace = "self", - [""] = "string", + [parser.FALLBACK] = "string", [1] = collect_fallback, }, 2)) --tseskipwhitespace["\n"] = setmetatable({[2] = countline, ["\r"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace}) @@ -366,7 +366,7 @@ do local tstring = selfify({}) tstring['\n'] = false tstring['\r'] = false - tstring[""] = "self" + tstring[parser.FALLBACK] = "self" tstring[1] = collect_fallback @@ -385,7 +385,7 @@ end do local tlongstring = {} defs.longstring = tlongstring - do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end}) + do local tllongstring_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end}) tllongstring_proper[1] = false -- placeholder for newline handling tllongstring_proper[2] = collect_fallback @@ -412,7 +412,7 @@ do local tlongstring = {} return "maybe_end" end end - tllmaybe_end[""] = "longstring_proper" + tllmaybe_end[parser.FALLBACK] = "longstring_proper" tllmaybe_end[1] = collect_fallback tllmaybe_end[-1] = function(state, token, rule) if not rule then @@ -473,6 +473,7 @@ mknewline(defs, 1) defs["-"] = "maybe_comment" do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs}) defs.maybe_comment = tmaybe_comment + tmaybe_comment[parser.EOZ] = "self" -- defs tmaybe_comment[-1] = function(state, token, rule) if rule ~= "comment" then state[#state+1] = "-" @@ -480,12 +481,12 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs}) end do local tmcomment = {comment_proper = selfify({})} tmaybe_comment.comment = tmcomment - tmcomment[""] = "comment_proper" + tmcomment[parser.FALLBACK] = "comment_proper" tmcomment["["] = "maybe_longcomment" mknewline(tmcomment, 1, defs) mknewline(tmcomment.comment_proper, 1, defs) - tmcomment.comment_proper[""] = "self" - do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end}) + tmcomment.comment_proper[parser.FALLBACK] = "self" + do local tllongcomment_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end}) tmcomment.longcomment = tllongcomment_proper do local tllmaybe_end = selfify({defs = defs}, "maybe_end") tllongcomment_proper.maybe_end = tllmaybe_end @@ -504,7 +505,7 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs}) return "maybe_end" end end - tllmaybe_end[""] = "longcomment_proper" + tllmaybe_end[parser.FALLBACK] = "longcomment_proper" tllmaybe_end[-1] = function(state, token, rule) if not rule then state.longcomment_close = nil @@ -543,6 +544,7 @@ end local STATE = parser.STATE defs.multitokens = setmetatable({ + [parser.EOZ] = "self", [-1] = function(state, token, rule) if not state[STATE].multitoken[token] then state[#state+1] = state[STATE].first @@ -736,18 +738,23 @@ function defs.string_open(state, token) assert("this shouldn't happen") end +local tokens = { + TK_AND = TK_AND, TK_BREAK = TK_BREAK, + TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION, + TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT, + TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE, + TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE, + TK_SHL = TK_SHL, TK_SHR = TK_SHR, + TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS, + TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING +} +for k,v in pairs(tokens) do + setmetatable(v, {__name=k, __tostring=function(self) return getmetatable(self).__name end}) +end + return { defs = defs, - tokens = { - TK_AND = TK_AND, TK_BREAK = TK_BREAK, - TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION, - TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT, - TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE, - TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE, - TK_SHL = TK_SHL, TK_SHR = TK_SHR, - TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS, - TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING - }, + tokens = tokens, reverse_keywords = reverse_keywords, reverse_tokens = { [TK_IDIV] = "//", [TK_CONCAT] = "..", [TK_DOTS] = "...", [TK_EQ] = "==", [TK_GE] = ">=", [TK_LE] = "<=", [TK_NE] = "~=", |