diff options
Diffstat (limited to 'printtokens.lua')
| -rw-r--r-- | printtokens.lua | 35 |
1 files changed, 35 insertions, 0 deletions
diff --git a/printtokens.lua b/printtokens.lua new file mode 100644 index 0000000..8f2d0f5 --- /dev/null +++ b/printtokens.lua @@ -0,0 +1,35 @@ +--[[ + This file is part of luatokens.lua - pure-Lua Lua tokenizer + Copyright (C) 2019 Soni L. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +--]] + +local parser = require "parser" + +do + local filename = arg[1] + local luatokens = require "luatokens" + local file = io.open(filename, "r") + local tokens = luatokens.defs + local state, err, etoken, estate = parser.parse(tokens, function() return file:read(8192) end) + if state then + for i,v in ipairs(state) do + v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v + print(i, v) -- TODO formatting + end + else + print("Parse error") + end +end |