1 files changed, 53 insertions, 19 deletions
diff --git a/luatokens.lua b/luatokens.lua
index 58a7d09..226a81a 100644
--- a/luatokens.lua
+++ b/luatokens.lua
@@ -99,17 +99,21 @@ defs.base = {
     ["Z"] = "alpha",
 }
 
-local function linecount(state, token, rule)
-    -- TODO fix
-    if token == "\n" or token == "\r" then
-        state.line = (state.line or 1) + 1
-    end
+local function countline(state, token, rule)
+    state.line = (state.line or 1) + 1
+end
+
+local function mknewline(t, hookn, fallback)
+    fallback = fallback or t
+    t["\n"] = setmetatable({[hookn] = countline, ["\r"] = setmetatable({}, {__index=fallback})}, {__index=fallback})
+    t["\r"] = setmetatable({[hookn] = countline, ["\n"] = setmetatable({}, {__index=fallback})}, {__index=fallback})
+    return t
 end
 
 do local tstring = selfify({})
     defs.string = tstring
     tstring.defs = defs
-    do local tsescapes = setmetatable({
+    do local tsescapes = setmetatable(mknewline({
             ["'"] = "insertraw",
             ['"'] = "insertraw",
             ['\\'] = "insertraw",
@@ -123,11 +127,10 @@ do local tstring = selfify({})
             ["z"] = "skipwhitespace",
             ["u"] = "unicode",
             ["x"] = "hex",
-            ["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
-            ["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
-            [1] = linecount,
-            [2] = function(state, token, rule) if token == "\r" or token == "\n" then collect_fallback(state, "\n") end end,
-        }, {__index = defs.base})
+            --["\n"] = setmetatable({[1] = countline, ["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
+            --["\r"] = setmetatable({[1] = countline, ["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
+            [1] = function(state, token, rule) if token == "\r" or token == "\n" then collect_fallback(state, "\n") end end,
+        }, 1, tstring), {__index = defs.base})
         defs.string.escapes = tsescapes
         tsescapes.string = defs.string
 
@@ -237,15 +240,16 @@ do local tstring = selfify({})
             tsescapes.unicode = tseunicode
         end
 
-        do local tseskipwhitespace = selfify({
+        do local tseskipwhitespace = selfify(mknewline({
                 string = defs.string,
                 whitespace = "self",
                 [""] = "string",
                 [1] = collect_fallback,
-                [2] = linecount,
-            })
+            }, 2))
+            --tseskipwhitespace["\n"] = setmetatable({[2] = countline, ["\r"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
+            --tseskipwhitespace["\r"] = setmetatable({[2] = countline, ["\n"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
             local tbase = defs.base
-            local tbasemap = {whitespace = "whitespace", newline = "whitespace"}
+            local tbasemap = {whitespace = "whitespace"}
             setmetatable(tseskipwhitespace, {__index = function(t, k) return tbasemap[tbase[k]] or tstring[k] end})
             tsescapes.skipwhitespace =  tseskipwhitespace
         end
@@ -276,21 +280,51 @@ do local tstring = selfify({})
     end
 end
 
-do local tlongstring = {}
+do local tlongstring = selfify({})
+    defs.longstring = tlongstring
     -- TODO
 end
 
 defs["'"] = "string_open"
 defs['"'] = "string_open"
 defs["["] = "maybe_longstring"
-defs[1] = linecount
+defs.maybe_longstring = setmetatable({
+    defs = defs,
+    ['['] = "longstring_open",
+    ['='] = "longstring_open",
+    longstring_count = selfify({
+        ["="] = function(state, token)
+            state.longstring_count = state.longstring_count + 1
+            return "self"
+        end,
+        longstring = defs.longstring
+    }),
+    longstring_open = function(state, token)
+        if token == "=" then
+            state.longstring_count = state.longstring_count or 0 + 1
+            return "longstring_count"
+        elseif token == "[" then
+            return "longstring"
+        end
+    end,
+    [-1] = function(state, token, rule)
+        if rule ~= "longstring_open" then
+            state[#state+1] = "["
+        end
+    end
+}, {__index=defs})
+
+-- these are needed for proper line counts
+--defs["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=defs})}, {__index=defs})
+--defs["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=defs})}, {__index=defs})
+mknewline(defs, 1)
 
-setmetatable(defs, {__index=whitespace})
+setmetatable(defs, {__index=defs.base})
 
 function defs.string_open(state, token)
     if not state.in_string then
         state[#state+1] = TK_STRING
-        state[COLLECT] = {}
+        state[COLLECT] = {coalesce=50} -- TODO tweak this for CPU/memory tradeoff?
         state.in_string = token
         return "string"
     end