Keywords and long strings

1 files changed, 101 insertions, 4 deletions

diff --git a/luatokens.lua b/luatokens.lua
index 226a81a..7bf9f68 100644
--- a/luatokens.lua
+++ b/luatokens.lua
@@ -9,12 +9,15 @@ local collect_fallback = parser.collect_fallback
 
 -- "dummies"
 -- see http://www.lua.org/source/5.3/llex.h.html#RESERVED
+-- keywords
 local TK_AND, TK_BREAK,
     TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
     TK_GOTO, TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
     TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
+    -- operators
     TK_IDIV, TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE,
     TK_SHL, TK_SHR,
+    -- misc
     TK_DBCOLON, TK_EOS,
     TK_FLT, TK_INT, TK_NAME, TK_STRING =
     {}, {},
@@ -26,7 +29,32 @@ local TK_AND, TK_BREAK,
     {}, {},
     {}, {}, {}, {}
 
-local defs = {}
+local keywords = {
+    ["and"] = TK_AND,
+    ["break"] = TK_BREAK,
+    ["do"] = TK_DO,
+    ["else"] = TK_ELSE,
+    ["elseif"] = TK_ELSEIF,
+    ["end"] = TK_END,
+    ["false"] = TK_FALSE,
+    ["for"] = TK_FOR,
+    ["function"] = TK_FUNCTION,
+    ["goto"] = TK_GOTO,
+    ["if"] = TK_IF,
+    ["in"] = TK_IN,
+    ["local"] = TK_LOCAL,
+    ["nil"] = TK_NIL,
+    ["not"] = TK_NOT,
+    ["or"] = TK_OR,
+    ["repeat"] = TK_REPEAT,
+    ["return"] = TK_RETURN,
+    ["then"] = TK_THEN,
+    ["true"] = TK_TRUE,
+    ["until"] = TK_UNTIL,
+    ["while"] = TK_WHILE,
+}
+
+local defs = selfify({})
 
 defs.base = {
     [" "] = "whitespace",
@@ -280,9 +308,46 @@ do local tstring = selfify({})
     end
 end
 
-do local tlongstring = selfify({})
+do local tlongstring = {}
     defs.longstring = tlongstring
-    -- TODO
+    do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
+        tllongstring_proper[1] = collect_fallback
+
+        do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
+            tllongstring_proper.maybe_end = tllmaybe_end
+            tllmaybe_end["="] = function(state, token)
+                state.longstring_close = state.longstring_close + 1
+                return "maybe_end"
+            end
+            tllmaybe_end["]"] = function(state, token)
+                if state.longstring_close == state.longstring_count then
+                    state.longstring_close = nil
+                    state.longstring_count = nil
+                    local pos = #state
+                    state[pos+1] = TK_STRING
+                    state[pos+2] = table.concat(state[COLLECT])
+                    state[COLLECT] = nil
+                    return "defs"
+                else
+                    collect_fallback(state, "]")
+                    collect_fallback(state, ("="):rep(state.longstring_close))
+                    state.longstring_close = 0
+                    return "maybe_end"
+                end
+            end
+            tllmaybe_end[-1] = function(state, token, rule)
+                if not rule then
+                    collect_fallback(state, "]")
+                    collect_fallback(state, ("="):rep(state.longstring_close))
+                    state.longstring_close = nil
+                end
+            end
+        end
+
+        tlongstring.longstring_proper = tllongstring_proper
+        mknewline(tlongstring, 1, tllongstring_proper)
+        setmetatable(tlongstring, {__index=tllongstring_proper})
+    end
 end
 
 defs["'"] = "string_open"
@@ -297,6 +362,10 @@ defs.maybe_longstring = setmetatable({
             state.longstring_count = state.longstring_count + 1
             return "self"
         end,
+        ["["] = function(state, token)
+            state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
+            return "longstring"
+        end,
         longstring = defs.longstring
     }),
     longstring_open = function(state, token)
@@ -304,6 +373,8 @@ defs.maybe_longstring = setmetatable({
             state.longstring_count = state.longstring_count or 0 + 1
             return "longstring_count"
         elseif token == "[" then
+            state.longstring_count = 0
+            state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
             return "longstring"
         end
     end,
@@ -319,12 +390,38 @@ defs.maybe_longstring = setmetatable({
 --defs["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=defs})}, {__index=defs})
 mknewline(defs, 1)
 
+defs.whitespace = "self"
+defs.hexdigit = "alpha"
+defs["_"] = "alpha"
+defs.in_alpha = setmetatable(selfify({digit = "in_alpha", hexdigit = "in_alpha", alpha = "in_alpha", _ = "in_alpha", [parser.EOZ] = "self"}, "in_alpha"), {__index=defs})
+function defs.alpha(state, token)
+    state[COLLECT] = {coalesce=15} -- TODO tweak this for CPU/memory tradeoff?
+    collect_fallback(state, token)
+    return "in_alpha"
+end
+defs.in_alpha[-1] = function(state, token, rule)
+    if rule == "alpha" or rule == "digit" or rule == "hexdigit" or token == "_" then
+        collect_fallback(state, token)
+    else
+        local key = table.concat(state[COLLECT])
+        state[COLLECT] = nil
+        local keyword = keywords[key]
+        if keyword then
+            state[#state+1] = keyword
+        else
+            local pos = #state
+            state[pos+1] = TK_NAME
+            state[pos+2] = key
+        end
+    end
+end
+
 setmetatable(defs, {__index=defs.base})
 
 function defs.string_open(state, token)
     if not state.in_string then
         state[#state+1] = TK_STRING
-        state[COLLECT] = {coalesce=50} -- TODO tweak this for CPU/memory tradeoff?
+        state[COLLECT] = {coalesce=63} -- TODO tweak this for CPU/memory tradeoff?
         state.in_string = token
         return "string"
     end