Add stuff that doesn't work

8 files changed, 396 insertions, 41 deletions

diff --git a/autotest.sh b/autotest.sh
new file mode 100755
index 0000000..54b998c
--- /dev/null
+++ b/autotest.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+until inotifywait -e modify compiler.lua testc.lua; do
+	date '+%s'
+	lua testc.lua
+done
diff --git a/compiler.lua b/compiler.lua
new file mode 100644
index 0000000..d67f9ec
--- /dev/null
+++ b/compiler.lua
@@ -0,0 +1,239 @@
+--[[
+    This file is part of cratera.lua - pure-Lua Cratera-to-Lua transpiler
+    Copyright (C) 2019  Soni L.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+--]]
+
+--[[
+    This software is based on Lua 5.1 and Lua 5.3
+
+    Lua 5.1 license:
+
+/******************************************************************************
+* Copyright (C) 1994-2012 Lua.org, PUC-Rio.  All rights reserved.
+*
+* Permission is hereby granted, free of charge, to any person obtaining
+* a copy of this software and associated documentation files (the
+* "Software"), to deal in the Software without restriction, including
+* without limitation the rights to use, copy, modify, merge, publish,
+* distribute, sublicense, and/or sell copies of the Software, and to
+* permit persons to whom the Software is furnished to do so, subject to
+* the following conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+
+    Lua 5.3 license:
+
+/******************************************************************************
+* Copyright (C) 1994-2018 Lua.org, PUC-Rio.
+*
+* Permission is hereby granted, free of charge, to any person obtaining
+* a copy of this software and associated documentation files (the
+* "Software"), to deal in the Software without restriction, including
+* without limitation the rights to use, copy, modify, merge, publish,
+* distribute, sublicense, and/or sell copies of the Software, and to
+* permit persons to whom the Software is furnished to do so, subject to
+* the following conditions:
+*
+* The above copyright notice and this permission notice shall be
+* included in all copies or substantial portions of the Software.
+*
+* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+******************************************************************************/
+--]]
+
+-- a parser.lua-based cratera compiler
+-- a few notes:
+-- * all "next" should be tables. as well as all "super" (which should be "next").
+--   (it wouldn't work properly without this)
+-- * when calling into a deeper level, remember to use the second return value "retry"
+--   (i.e. set it to true)
+
+local parser = require "parser"
+local selfify = parser.selfify
+local STATE = parser.STATE
+local l = require "luatokens".tokens
+local assert, type, setmetatable = assert, type, setmetatable
+
+local function tostring__name(self)
+    return getmetatable(self).__name
+end
+
+local function Upvaldesc() return {
+    name = nil, -- TString -- upvalue name (for debug information)
+    instack = false, -- lu_byte -- whether it is in stack (register)
+    idx = 0, -- lu_byte -- index of upvalue (in stack or in outer function's list)
+} end
+local function LocVar() return {
+    varname = nil, -- TString
+    startpc = 0, -- int -- first point where variable is active
+    endpc = 0, -- int -- first point where variable is dead
+} end
+local function Proto() return { -- is a GC object
+    numparams = 0, -- lu_byte -- number of fixed parameters
+    is_vararg = false, -- lu_byte but boolean
+    maxstacksize = 0, -- lu_byte -- number of registers needed by this function
+    k = {}, -- TValue * -- constants used by the function
+    code = {}, -- Instruction * -- opcodes
+    p = {}, -- Proto ** -- functions defined inside the function
+    lineinfo = {}, -- int * -- map from opcodes to source lines (debug information)
+    locvars = {}, -- LocVar * -- information about local variables (debug information)
+    uvalues = {}, -- Upvaldesc * -- upvalue information
+} end
+local function FuncState() return {
+    f = nil, -- Proto -- current function header
+    prev = nil, -- FuncState -- enclosing function
+    ls = nil, -- LexState -- lexical state
+    bl = nil, -- BlockCnt -- chain of current blocks
+    pc = 0, -- int -- next position to code (equivalent to 'ncode')
+    lasttarget = 0, -- int -- 'label' of last 'jump label'
+    jpc = 0, -- int -- number of pending jumps to 'pc'
+    --nk = 0, -- int -- number of elements in 'k'
+    --np = 0, -- int -- number of elements in 'p'
+    firstlocal = 0, -- int -- index of first local var (in Dyndata array)
+    nlocvars = 0, -- short -- number of elements in 'f->locvars'
+    nactvar = 0, -- lu_byte -- number of active local variables
+    nups = 0, -- lu_byte -- number of upvalues
+    freereg = 0, -- lu_byte -- first free register
+} end
+local function Labeldesc() return {
+    name = nil, -- TString -- label identifier
+    pc = nil, -- int -- position in code
+    line = nil, -- int -- line where it appeared
+    nactvar = nil, -- lu_byte -- local level where it appears in current block
+} end
+local function Dyndata() return {
+    actvar = {}, -- ArrayList of Vardesc (short) -- list of active local variables
+    gt = {}, -- Labellist (ArrayList of Labeldesc) -- list of pending gotos
+    label = {}, -- Labellist (ArrayList of Labeldesc) -- list of active labels
+} end
+local function ParserState() return { -- LexState
+    fs = nil, -- FuncState *
+    dyd = nil, -- Dyndata *
+} end
+
+local gotostatname = {[parser.EOZ] = false}
+local gotostatnamemt = {__index=gotostatname, __name="gotostatname", __tostring=tostring__name}
+gotostatname[parser.FALLBACK] = function(state, token)
+    assert(type(token) == "string")
+    state[#state+1] = "goto"
+    state[#state+1] = token
+    return state[STATE].next
+end
+
+local gotostat = {[parser.EOZ] = false}
+local gotostatmt = {__index=gotostat, __name="gotostat", __tostring=tostring__name}
+gotostat[l.TK_NAME] = function(state, token)
+    return setmetatable({next = state[STATE].next}, gotostatnamemt)
+end
+
+local singlevar = {[parser.EOZ] = false}
+local singlevarmt = {__index=singlevar, __name="singlevar", __tostring=tostring__name}
+singlevar[parser.FALLBACK] = function(state, token)
+    assert(type(token) == "string")
+    state[#state+1] = token
+    return state[STATE].next
+end
+
+local primaryexp = {[parser.EOZ] = false}
+local primaryexpmt = {__name="primaryexp", __tostring=tostring__name}
+primaryexp['('] = function(state, token) end
+primaryexp[l.TK_NAME] = function(state, token)
+    return setmetatable({next=state[STATE].next}, singlevarmt)
+end
+
+local suffixedexp = {}
+local suffixedexpmt = {__name="suffixedexp", __tostring=tostring__name}
+suffixedexp.next = function() end
+
+local exprstat = {}
+local exprstatmt = {__index=exprstat, __name="exprstat", __tostring=tostring__name}
+exprstat.next = {}
+
+local statementt = {[parser.EOZ] = false}
+local statementmt = {__index=statementt, __name="statement", __tostring=tostring__name}
+local function statement(state, token)
+    local cur = state[STATE]
+    return setmetatable({next = cur.next}, statementmt), true
+end
+statementt[";"] = function(state, token)
+    state[#state+1] = token
+    return "next"
+end
+statementt[l.TK_IF] = function(state, token) end
+statementt[l.TK_WHILE] = function(state, token) end
+statementt[l.TK_DO] = function(state, token) end
+statementt[l.TK_FOR] = function(state, token) end
+statementt[l.TK_REPEAT] = function(state, token) end
+statementt[l.TK_FUNCTION] = function(state, token) end
+statementt[l.TK_LOCAL] = function(state, token) end
+statementt[l.TK_DBCOLON] = function(state, token) end
+statementt[l.TK_RETURN] = function(state, token) end
+statementt[l.TK_BREAK] = function(state, token)
+    state[#state+1] = "break"
+    return "next"
+end
+statementt[l.TK_GOTO] = function(state, token)
+    return setmetatable({next = state[STATE].next}, gotostatmt)
+end
+statementt[parser.FALLBACK] = function(state, token)
+    return setmetatable({super = state[STATE].next}, exprstatmt), true
+end
+
+local statlistt = {}
+local statlistmt = {__index=statlistt, __name="statlist", __tostring=tostring__name}
+local function statlist(state, token)
+    local cur = state[STATE]
+    return setmetatable(selfify({super = cur.next, withuntil = cur.withuntil}, "next"), statlistmt), true
+end
+statlistt[l.TK_ELSE] = function() return "super", true end
+statlistt[l.TK_ELSEIF] = function() return "super", true end
+statlistt[l.TK_END] = function() return "super", true end
+statlistt[parser.EOZ] = function() return "super", true end
+statlistt[l.TK_UNTIL] = function() return "withuntil", true end
+statlistt[parser.FALLBACK] = statement
+
+local mainfunc = setmetatable({}, {__name="mainfunc", __tostring=tostring__name})
+mainfunc.withuntil = "super"
+mainfunc[parser.EOZ] = parser.FALLBACK
+mainfunc[parser.FALLBACK] = statlist
+mainfunc.next = {
+    [parser.EOZ] = {}
+}
+
+local defs = setmetatable({}, {__name="defs", __tostring=tostring__name})
+defs[parser.EOZ] = parser.FALLBACK
+defs[parser.FALLBACK] = function(state, token) return mainfunc, true end
+
+
+return {
+    defs = defs,
+}
diff --git a/cratera.lua b/cratera.lua
new file mode 100644
index 0000000..fabb371
--- /dev/null
+++ b/cratera.lua
@@ -0,0 +1,71 @@
+--[[
+    cratera.lua - pure-Lua Cratera-to-Lua transpiler
+    Copyright (C) 2019  Soni L.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+--]]
+
+-- This code is highly experimental and not very good
+
+local parser = require "parser"
+local luatokens = require "luatokens"
+local compiler = require "compiler"
+
+local CRATERA_SEED = nil -- TODO
+
+local function cratera_load(reader)
+    local f, s, i = parser.stream(luatokens.defs, reader)
+    local nl = 1
+    local otherstate = {}
+    local f, s, i = parser.stream(compiler.defs, function()
+        local tokens
+        repeat
+            local pos, state, transemsg, etoken, estate = f(s, i)
+            otherstate.line = state.line
+            i = pos
+            if not i then return nil end
+            if not state then error(transemsg) end
+            tokens = {}
+            for i,v in ipairs(state) do
+                state[i] = nil
+                tokens[i] = v
+            end
+        until #tokens > 0 or not transemsg
+        return tokens
+    end, otherstate)
+    local function fn()
+        function fn()
+            local tokens
+            repeat
+                local pos, state, transemsg, etoken, estate, est = f(s, i)
+                i = pos
+                if not i then return nil end
+                if not state then error(transemsg .. " " .. tostring(etoken)) end
+                tokens = {""}
+                for i,v in ipairs(state) do
+                    state[i] = nil
+                    tokens[i+1] = v
+                end
+            until #tokens > 1 or not transemsg
+            return table.concat(tokens, " ")
+        end
+        local ret = fn()
+        return string.sub(ret, 2)
+    end
+    return load(function()
+        return fn()
+    end)
+end
+
+return {load = cratera_load, CRATERA_SEED = CRATERA_SEED}
diff --git a/luatokens.lua b/luatokens.lua
index ecc3b6d..9ca19ac 100644
--- a/luatokens.lua
+++ b/luatokens.lua
@@ -272,7 +272,7 @@ do local tstring = selfify({})
             end
             return "string"
         end
-        tsescapes.digitc = setmetatable(selfify({[""] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring})
+        tsescapes.digitc = setmetatable(selfify({[parser.FALLBACK] = tsescapes.digit, string = tstring}, "digitc"), {__index=tstring})
         tsescapes.digitc[1]=function(state, token, rule)
             if rule == nil then
                 collect_fallback(state, string.char(state.in_digit))
@@ -346,7 +346,7 @@ do local tstring = selfify({})
         do local tseskipwhitespace = selfify(mknewline({
                 string = defs.string,
                 whitespace = "self",
-                [""] = "string",
+                [parser.FALLBACK] = "string",
                 [1] = collect_fallback,
             }, 2))
             --tseskipwhitespace["\n"] = setmetatable({[2] = countline, ["\r"] = setmetatable({}, {__index=tseskipwhitespace})}, {__index=tseskipwhitespace})
@@ -366,7 +366,7 @@ do local tstring = selfify({})
     tstring['\n'] = false
     tstring['\r'] = false
 
-    tstring[""] = "self"
+    tstring[parser.FALLBACK] = "self"
 
     tstring[1] = collect_fallback
 
@@ -385,7 +385,7 @@ end
 
 do local tlongstring = {}
     defs.longstring = tlongstring
-    do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
+    do local tllongstring_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
         tllongstring_proper[1] = false -- placeholder for newline handling
         tllongstring_proper[2] = collect_fallback
 
@@ -412,7 +412,7 @@ do local tlongstring = {}
                     return "maybe_end"
                 end
             end
-            tllmaybe_end[""] = "longstring_proper"
+            tllmaybe_end[parser.FALLBACK] = "longstring_proper"
             tllmaybe_end[1] = collect_fallback
             tllmaybe_end[-1] = function(state, token, rule)
                 if not rule then
@@ -473,6 +473,7 @@ mknewline(defs, 1)
 defs["-"] = "maybe_comment"
 do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
     defs.maybe_comment = tmaybe_comment
+    tmaybe_comment[parser.EOZ] = "self" -- defs
     tmaybe_comment[-1] = function(state, token, rule)
         if rule ~= "comment" then
             state[#state+1] = "-"
@@ -480,12 +481,12 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
     end
     do local tmcomment = {comment_proper = selfify({})}
         tmaybe_comment.comment = tmcomment
-        tmcomment[""] = "comment_proper"
+        tmcomment[parser.FALLBACK] = "comment_proper"
         tmcomment["["] = "maybe_longcomment"
         mknewline(tmcomment, 1, defs)
         mknewline(tmcomment.comment_proper, 1, defs)
-        tmcomment.comment_proper[""] = "self"
-        do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
+        tmcomment.comment_proper[parser.FALLBACK] = "self"
+        do local tllongcomment_proper = selfify({[parser.FALLBACK] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
             tmcomment.longcomment = tllongcomment_proper
             do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
                 tllongcomment_proper.maybe_end = tllmaybe_end
@@ -504,7 +505,7 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
                         return "maybe_end"
                     end
                 end
-                tllmaybe_end[""] = "longcomment_proper"
+                tllmaybe_end[parser.FALLBACK] = "longcomment_proper"
                 tllmaybe_end[-1] = function(state, token, rule)
                     if not rule then
                         state.longcomment_close = nil
@@ -543,6 +544,7 @@ end
 local STATE = parser.STATE
 
 defs.multitokens = setmetatable({
+    [parser.EOZ] = "self",
     [-1] = function(state, token, rule)
         if not state[STATE].multitoken[token] then
             state[#state+1] = state[STATE].first
@@ -736,18 +738,23 @@ function defs.string_open(state, token)
     assert("this shouldn't happen")
 end
 
+local tokens = {
+    TK_AND = TK_AND, TK_BREAK = TK_BREAK,
+    TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION,
+    TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT,
+    TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE,
+    TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE,
+    TK_SHL = TK_SHL, TK_SHR = TK_SHR,
+    TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS,
+    TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING
+}
+for k,v in pairs(tokens) do
+    setmetatable(v, {__name=k, __tostring=function(self) return getmetatable(self).__name end})
+end
+
 return {
     defs = defs,
-    tokens = {
-        TK_AND = TK_AND, TK_BREAK = TK_BREAK,
-        TK_DO = TK_DO, TK_ELSE = TK_ELSE, TK_ELSEIF = TK_ELSEIF, TK_END = TK_END, TK_FALSE = TK_FALSE, TK_FOR = TK_FOR, TK_FUNCTION = TK_FUNCTION,
-        TK_GOTO = TK_GOTO, TK_IF = TK_IF, TK_IN = TK_IN, TK_LOCAL = TK_LOCAL, TK_NIL = TK_NIL, TK_NOT = TK_NOT, TK_OR = TK_OR, TK_REPEAT = TK_REPEAT,
-        TK_RETURN = TK_RETURN, TK_THEN = TK_THEN, TK_TRUE = TK_TRUE, TK_UNTIL = TK_UNTIL, TK_WHILE = TK_WHILE,
-        TK_IDIV = TK_IDIV, TK_CONCAT = TK_CONCAT, TK_DOTS = TK_DOTS, TK_EQ = TK_EQ, TK_GE = TK_GE, TK_LE = TK_LE, TK_NE = TK_NE,
-        TK_SHL = TK_SHL, TK_SHR = TK_SHR,
-        TK_DBCOLON = TK_DBCOLON, TK_EOS = TK_EOS,
-        TK_FLT = TK_FLT, TK_INT = TK_INT, TK_NAME = TK_NAME, TK_STRING = TK_STRING
-    },
+    tokens = tokens,
     reverse_keywords = reverse_keywords,
     reverse_tokens = {
         [TK_IDIV] = "//", [TK_CONCAT] = "..", [TK_DOTS] = "...", [TK_EQ] = "==", [TK_GE] = ">=", [TK_LE] = "<=", [TK_NE] = "~=",
diff --git a/parser.lua b/parser.lua
index 7410571..34bfce2 100644
--- a/parser.lua
+++ b/parser.lua
@@ -16,34 +16,43 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 --]]
 
+local function ts(self) return getmetatable(self).__name end
+
 -- key for STATE
-local STATE = {}
+local STATE = setmetatable({}, {__name="STATE", __tostring=ts})
 -- key for DATA
-local DATA = {}
+local DATA = setmetatable({}, {__name="DATA", __tostring=ts})
 -- key for GENERATOR
-local GEN = {}
+local GEN = setmetatable({}, {__name="GEN", __tostring=ts})
 -- key for DATA OFFSET
-local OFFDATA = {}
+local OFFDATA = setmetatable({}, {__name="OFFDATA", __tostring=ts})
 -- key for End of Stream
-local EOZ = {}
+local EOZ = setmetatable({}, {__name="EOZ", __tostring=ts})
+-- key for number rules (prevent conflict with hooks)
+local NUMBER = setmetatable({}, {__name="NUMBER", __tostring=ts})
+-- key for fallback rules (prevent conflict with empty string)
+local FALLBACK = setmetatable({}, {__name="FALLBACK", __tostring=ts})
 
 local optimize_lookups = {}
 for i=0, 255 do
     optimize_lookups[i] = string.char(i)
 end
 
-local type, tostring
-    = type, tostring
+local type, tostring, string_byte
+    = type, tostring, string.byte
 
 local function get_next_common(state, in_pos, token)
     -- note: must preserve "token" - do not call recursively with a different token
-    local transition
-    if state[STATE] then
-        local st = state[STATE]
+    local transition, retry
+    local st = state[STATE]
+    if st then
         local rule = st[token]
         if not rule and token == EOZ then
             return in_pos, state
         end
+        if type(token) == "number" then
+            rule = st[NUMBER]
+        end
         do -- pre-hooks
             local pos = -1
             local hook = st[pos]
@@ -57,7 +66,7 @@ local function get_next_common(state, in_pos, token)
         end
         transition = rule
         if transition == nil then
-            transition = st[""]
+            transition = st[FALLBACK]
         end
         local recheck = true
         while recheck do
@@ -67,7 +76,10 @@ local function get_next_common(state, in_pos, token)
                 transition = st[transition]
                 recheck = true
             elseif tytrans == "function" then
-                transition = transition(state, token)
+                transition, retry = transition(state, token)
+                recheck = true
+            elseif tytrans == "table" and st[transition] ~= nil then
+                transition = st[transition]
                 recheck = true
             end
         end
@@ -88,8 +100,9 @@ local function get_next_common(state, in_pos, token)
     if not state[STATE] then
         -- unexpected token. stream consumer may attempt to recover,
         -- but we do this mostly to differentiate it from "end of stream" condition.
-        return in_pos - 1, nil, "unexpected token", token, state
+        return in_pos - 1, nil, "unexpected token", token, state, st
     end
+    if retry then in_pos = in_pos - 1 end
     return in_pos, state, transition -- TODO is this what we should be returning?
 end
 
@@ -120,7 +133,7 @@ local function get_next_string(state, in_pos)
         end
     end
     in_pos = in_pos + 1
-    local token = optimize_lookups[string.byte(state[DATA], in_pos - state[OFFDATA], in_pos - state[OFFDATA])]
+    local token = optimize_lookups[string_byte(state[DATA], in_pos - state[OFFDATA], in_pos - state[OFFDATA])]
     if token == nil then
         state[OFFDATA] = in_pos - 1
         state[DATA] = state[GEN]()
@@ -129,8 +142,8 @@ local function get_next_string(state, in_pos)
     return get_next_common(state, in_pos, token)
 end
 
-local function stream(defs, data)
-    local state = {}
+local function stream(defs, data, state)
+    local state = state or {}
     local fn
     state[STATE] = defs
     if type(data) == "function" then
@@ -145,8 +158,8 @@ local function stream(defs, data)
     return fn, state, state[OFFDATA]
 end
 
-local function parse(defs, data)
-    for pos, state, transemsg, etoken, estate in stream(defs, data) do
+local function parse(defs, data, state)
+    for pos, state, transemsg, etoken, estate in stream(defs, data, state) do
         if not state then
             -- parse error
             return nil, transemsg, etoken, estate
@@ -165,6 +178,8 @@ return {
     STATE = STATE,
     COLLECT = COLLECT,
     EOZ = EOZ,
+    FALLBACK = FALLBACK,
+    NUMBER = NUMBER,
     stream = stream,
     parse = parse,
     -- common utility function
diff --git a/printtokens.lua b/printtokens.lua
index 0b3b5b5..62e8fd9 100644
--- a/printtokens.lua
+++ b/printtokens.lua
@@ -28,7 +28,7 @@ defs['-'] = function(state, token)
     state.file = io.stdin
     return "self"
 end
-defs[""] = function(state, token)
+defs[parser.FALLBACK] = function(state, token)
     if state.filename then
         error("Must specify only one filename")
     end
@@ -47,7 +47,7 @@ defs[-1] = function(state, token, rule)
         error("Unknown option: " .. token)
     end
 end
-defs['--'] = parser.selfify({[""] = defs[""], [parser.EOZ] = defs[parser.EOZ]})
+defs['--'] = parser.selfify({[parser.FALLBACK] = defs[parser.FALLBACK], [parser.EOZ] = defs[parser.EOZ]})
 
 local state = parser.parse(defs, arg)
 local luatokens = require "luatokens"
diff --git a/testc.lua b/testc.lua
new file mode 100644
index 0000000..1c8f572
--- /dev/null
+++ b/testc.lua
@@ -0,0 +1,17 @@
+local function printr(...)
+    print(...)
+    return ...
+end
+
+local realload = load
+load = function(target, ...)
+    if type(target) == "function" then
+        return realload(function() return printr(target()) end, ...)
+    else
+        return realload(printr(target), ...)
+    end
+end
+
+local cratera = require "cratera"
+
+assert(printr(cratera.load("")))()
diff --git a/test.lua b/testp.lua
index 9ad6aa0..44bd5a3 100644
--- a/test.lua
+++ b/testp.lua
@@ -42,7 +42,7 @@ do -- trim left spaces
     defs['\f'] = "whitespace"
     defs['\v'] = "whitespace"
     defs.whitespace = "self"
-    defs[''] = function(state, token)
+    defs[parser.FALLBACK] = function(state, token)
         state[#state + 1] = token
         if #state > 20 then
             state[1] = table.concat(state)
@@ -54,7 +54,7 @@ do -- trim left spaces
     end
     defs.start = {}
     defs.start.self = defs.start
-    defs.start[''] = function(state, token)
+    defs.start[parser.FALLBACK] = function(state, token)
         state[#state + 1] = token
         if #state > 20 then
             state[1] = table.concat(state)