It's working better now...

1 files changed, 15 insertions, 15 deletions

diff --git a/luatokens.lua b/luatokens.lua
index 81cbc11..2ac2cc3 100644
--- a/luatokens.lua
+++ b/luatokens.lua
@@ -3,6 +3,9 @@
 -- we need some stuff from here
 local parser = require "parser"
 local selfify = parser.selfify
+local EOF = parser.EOF
+local COLLECT = parser.COLLECT
+local collect_fallback = parser.collect_fallback
 
 -- "dummies"
 local TK_STRING = {}
@@ -106,25 +109,25 @@ do local tstring = selfify({})
             ["\n"] = setmetatable({["\r"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
             ["\r"] = setmetatable({["\n"] = setmetatable({}, {__index=tstring})}, {__index=tstring}),
             [1] = linecount,
-            [2] = print
         }, {__index = tokens.base})
         tokens.string.escapes = tsescapes
         tsescapes.string = tokens.string
 
         function tsescapes.insertraw(state, token)
-            state[#state+1] = token
+            collect_fallback(state, token)
             return "string"
         end
 
         do
             local map = { ["a"] = "\a", ["b"] = "\b", ["f"] = "\f", ["n"] = "\n", ["r"] = "\r", ["t"] = "\t", ["v"] = "\v" }
             function tsescapes.insertmap(state, token)
-                state[#state+1] = map[token]
+                collect_fallback(state, map[token])
                 return "string"
             end
         end
 
         function tsescapes.digit(state, token)
+            print(state, token)
             local digit = string.find("1234567890", token, 1, true)
             local num = state.in_digit
             if digit then
@@ -138,21 +141,21 @@ do local tstring = selfify({})
             if num > 255 then
                 return nil
             end
-            state[#state+1] = string.char(num)
+            collect_fallback(state, string.char(num))
             state.in_digit = nil
             state.c = nil
             return "string"
         end
         tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, digitc = "self", string = tstring}), {__index=tstring})
 
-        tsescapes.hex = setmetatable(selfify({string = tokens.string}), {__index=tokens.base})
+        tsescapes.hex = setmetatable(selfify({string = tokens.string, digit = "hexdigit"}), {__index=tokens.base})
         function tsescapes.hex.hexdigit(state, token)
             local digit = string.find("123456789ABCDEF0123456789abcdef0", token, 1, true)
             assert(digit, "this should never be called for non-hex-digits")
             local num = state.in_hex
             if num then
                 num = num * 16 + digit % 16
-                state[#state+1] = string.char(num)
+                collect_fallback(state, string.char(num))
                 state.in_hex = nil
                 return "string"
             else
@@ -165,7 +168,7 @@ do local tstring = selfify({})
                 string = tokens.string,
                 whitespace = "self",
                 [""] = "string",
-                [1] = parser.insert_fallback,
+                [1] = collect_fallback,
                 [2] = linecount,
             })
             local tbase = tokens.base
@@ -185,17 +188,13 @@ do local tstring = selfify({})
 
     tstring[""] = "self"
 
-    tstring[1] = parser.insert_fallback
+    tstring[1] = collect_fallback
 
     function tstring.close(state, token)
         if state.in_string == token then
-            local i = state.string_start
             state.in_string = nil
-            state.string_start = nil
-            state[i+1] = table.concat(state, '', i+1)
-            for j=i+2, #state do
-                state[j]=nil
-            end
+            state[#state+1] = table.concat(state[COLLECT])
+            state[COLLECT] = nil
             return "tokens"
         else
             state[#state+1] = token
@@ -206,14 +205,15 @@ end
 
 tokens["'"] = "string_open"
 tokens['"'] = "string_open"
+tokens[1] = linecount
 
 setmetatable(tokens, {__index=whitespace})
 
 function tokens.string_open(state, token)
     if not state.in_string then
         state[#state+1] = TK_STRING
+        state[COLLECT] = {}
         state.in_string = token
-        state.string_start = #state
         return "string"
     end
     assert("this shouldn't happen")