From 726677253dffe0c8ea916785863111cc96724967 Mon Sep 17 00:00:00 2001 From: Stephen Leitnick Date: Wed, 26 Sep 2018 15:17:58 -0400 Subject: [PATCH] Updated plugin --- plugin/AeroGameFrameworkPlugin.rbxmx | 946 +++++++++++---------------- 1 file changed, 366 insertions(+), 580 deletions(-) diff --git a/plugin/AeroGameFrameworkPlugin.rbxmx b/plugin/AeroGameFrameworkPlugin.rbxmx index bf13612..ccf29bb 100644 --- a/plugin/AeroGameFrameworkPlugin.rbxmx +++ b/plugin/AeroGameFrameworkPlugin.rbxmx @@ -2,12 +2,12 @@ false null nil - + AeroGameFrameworkPlugin - + false @@ -626,7 +626,7 @@ function SetupCode() guiCode.ScrollingFrame.Content.Code:ClearAllChildren() StartNewLine() - for token,src in Lexer.lua(code, {}, {}) do + for token,src in Lexer.scan(code) do src = tostring(src):gsub("\t", " ") if (token == "space") then local newLines = select(2, src:gsub("\n", "\n")) @@ -932,7 +932,7 @@ pluginGui:GetPropertyChangedSignal("Enabled"):Connect(function() end)]]> - + Item @@ -1307,7 +1307,7 @@ return Item]]> - + true @@ -1331,7 +1331,7 @@ return __MODULENAME]]> - + true @@ -1369,7 +1369,7 @@ return __MODULENAME]]> - + true @@ -1412,7 +1412,7 @@ return __MODULENAME]]> - + true @@ -1446,7 +1446,7 @@ return __MODULENAME]]> - + UpdateChecker @@ -1501,7 +1501,7 @@ end return UpdateChecker]]> - + Version @@ -1584,7 +1584,7 @@ return Version]]> - + Theme @@ -1711,484 +1711,7 @@ return Theme]]> - - - - Lexer - {29AB533B-B989-4EBA-99E0-AC0ACF9F6C78} - s = "for i=1,n do" --- > for t,v in lexer.lua(s) do print(t,v) end --- keyword for --- iden i --- = = --- number 1 --- , , --- iden n --- keyword do --- --- See the Guide for further @{06-data.md.Lexical_Scanning|discussion} --- @module pl.lexer - -local yield, wrap = coroutine.yield, coroutine.wrap -local strfind = string.find -local strsub = string.sub -local append = table.insert - -local function assert_arg(idx, val, tp) - if (type(val) ~= tp) then - error("argument " .. idx .. " must be " .. tp, 2) - end -end - -local lexer = {} - -local NUMBER1 = "^[%+%-]?%d+%.?%d*[eE][%+%-]?%d+" -local NUMBER2 = "^[%+%-]?%d+%.?%d*" -local NUMBER3 = "^0x[%da-fA-F]+" -local NUMBER4 = "^%d+%.?%d*[eE][%+%-]?%d+" -local NUMBER5 = "^%d+%.?%d*" -local IDEN = "^[%a_][%w_]*" -local WSPACE = "^%s+" -local STRING1 = "^(['\"])%1" -- empty string -local STRING2 = [[^(['"])(\*)%2%1]] -local STRING3 = [[^(['"]).-[^\](\*)%2%1]] -local CHAR1 = "^''" -local CHAR2 = [[^'(\*)%1']] -local CHAR3 = [[^'.-[^\](\*)%1']] -local PREPRO = "^#.-[^\\]\n" - -local plain_matches, lua_matches, cpp_matches, lua_keyword, cpp_keyword, lua_builtin - -local function tdump(tok) - return yield(tok,tok) -end - -local function ndump(tok, options) - if (options and options.number) then - tok = tonumber(tok) - end - return yield("number", tok) -end - --- regular strings, single or double quotes; usually we want them --- without the quotes -local function sdump(tok, options) - if (options and options.string) then - tok = tok:sub(2, -2) - end - return yield("string", tok) -end - --- long Lua strings need extra work to get rid of the quotes -local function sdump_l(tok, options, findres) - if (options and options.string) then - local quotelen = 3 - if (findres[3]) then - quotelen = (quotelen + findres[3]:len()) - end - tok = tok:sub(quotelen, -quotelen) - if (tok:sub(1, 1) == "\n") then - tok = tok:sub(2) - end - end - return yield("string", tok) -end - -local function chdump(tok, options) - if (options and options.string) then - tok = tok:sub(2, -2) - end - return yield("char", tok) -end - -local function cdump(tok) - return yield("comment", tok) -end - -local function wsdump(tok) - return yield("space", tok) -end - -local function pdump(tok) - return yield("prepro", tok) -end - -local function plain_vdump(tok) - return yield("iden", tok) -end - -local function lua_vdump(tok) - if (lua_keyword[tok]) then - return yield("keyword", tok) - elseif (lua_builtin[tok]) then - return yield("builtin", tok) - else - return yield("iden", tok) - end -end - -local function cpp_vdump(tok) - if (cpp_keyword[tok]) then - return yield("keyword", tok) - else - return yield("iden", tok) - end -end - ---- create a plain token iterator from a string or file-like object. --- @tparam string|file s a string or a file-like object with `:read()` method returning lines. --- @tab matches an optional match table - array of token descriptions. --- A token is described by a `{pattern, action}` pair, where `pattern` should match --- token body and `action` is a function called when a token of described type is found. --- @tab[opt] filter a table of token types to exclude, by default `{space=true}` --- @tab[opt] options a table of options; by default, `{number=true,string=true}`, --- which means convert numbers and strip string quotes. -function lexer.scan(s, matches, filter, options) - local file = (type(s) ~= "string" and s) - filter = (filter or {space = true}) - options = (options or {number = true, string = true}) - if (filter) then - if (filter.space) then - filter[wsdump] = true - end - if (filter.comments) then - filter[cdump] = true - end - end - if (not matches) then - if (not plain_matches) then - plain_matches = { - {WSPACE, wsdump}, - {NUMBER3, ndump}, - {IDEN, plain_vdump}, - {NUMBER1, ndump}, - {NUMBER2, ndump}, - {STRING1, sdump}, - {STRING2, sdump}, - {STRING3, sdump}, - {"^.", tdump} - } - end - matches = plain_matches - end - local function lex(first_arg) - local line_nr = 0 - local next_line = (file and file:read()) - local sz = (file and 0 or #s) - local idx = 1 - - -- res is the value used to resume the coroutine. - local function handle_requests(res) - while (res) do - local tp = type(res) - -- insert a token list - if (tp == "table") then - res = yield("", "") - for _,t in ipairs(res) do - res = yield(t[1], t[2]) - end - elseif (tp == "string") then -- or search up to some special pattern - local i1, i2 = strfind(s, res, idx) - if (i1) then - local tok = strsub(s, i1, i2) - idx = (i2 + 1) - res = yield("", tok) - else - res = yield("", "") - idx = (sz + 1) - end - else - res = yield(line_nr, idx) - end - end - end - - handle_requests(first_arg) - if (not file) then - line_nr = 1 - end - - while (true) do - if (idx > sz) then - if (file) then - if (not next_line) then return end - s = next_line - line_nr = (line_nr + 1) - next_line = file:read() - if next_line then - s = (s .. "\n") - end - idx, sz = 1, #s - else - while (true) do - handle_requests(yield()) - end - end - end - - for _,m in ipairs(matches) do - local pat = m[1] - local fun = m[2] - local findres = {strfind(s, pat, idx)} - local i1, i2 = findres[1], findres[2] - if (i1) then - local tok = strsub(s, i1, i2) - idx = (i2 + 1) - local res - if (not (filter and filter[fun])) then - lexer.finished = (idx > sz) - res = fun(tok, options, findres) - end - if (not file and tok:find("\n")) then - -- Update line number. - local _, newlines = tok:gsub("\n", {}) - line_nr = (line_nr + newlines) - end - handle_requests(res) - break - end - end - end - end - return wrap(lex) -end - -local function isstring(s) - return (type(s) == "string") -end - ---- insert tokens into a stream. --- @param tok a token stream --- @param a1 a string is the type, a table is a token list and --- a function is assumed to be a token-like iterator (returns type & value) --- @string a2 a string is the value -function lexer.insert(tok, a1, a2) - if (not a1) then return end - local ts - if (isstring(a1) and isstring(a2)) then - ts = {{a1,a2}} - elseif (type(a1) == "function") then - ts = {} - for t,v in a1() do - append(ts, {t, v}) - end - else - ts = a1 - end - tok(ts) -end - ---- get everything in a stream upto a newline. --- @param tok a token stream --- @return a string -function lexer.getline(tok) - local _,v = tok(".-\n") - return v -end - ---- get current line number. --- @param tok a token stream --- @return the line number. --- if the input source is a file-like object, --- also return the column. -function lexer.lineno(tok) - return tok(0) -end - ---- get the rest of the stream. --- @param tok a token stream --- @return a string -function lexer.getrest(tok) - local _,v = tok(".+") - return v -end - ---- get the Lua keywords as a set-like table. --- So `res["and"]` etc would be `true`. --- @return a table -function lexer.get_keywords() - if (not lua_keyword) then - lua_keyword = { - ["and"] = true, ["break"] = true, ["do"] = true, - ["else"] = true, ["elseif"] = true, ["end"] = true, - ["false"] = true, ["for"] = true, ["function"] = true, - ["if"] = true, ["in"] = true, ["local"] = true, ["nil"] = true, - ["not"] = true, ["or"] = true, ["repeat"] = true, - ["return"] = true, ["then"] = true, ["true"] = true, - ["until"] = true, ["while"] = true, ["self"] = true - } - end - return lua_keyword -end - - -function lexer.get_builtins() - if (not lua_builtin) then - lua_builtin = { - ["assert"] = true;["collectgarbage"] = true;["error"] = true;["_G"] = true; - ["gcinfo"] = true;["getfenv"] = true;["getmetatable"] = true;["ipairs"] = true; - ["loadstring"] = true;["newproxy"] = true;["next"] = true;["pairs"] = true; - ["pcall"] = true;["print"] = true;["rawequal"] = true;["rawget"] = true;["rawset"] = true; - ["select"] = true;["setfenv"] = true;["setmetatable"] = true;["tonumber"] = true; - ["tostring"] = true;["type"] = true;["unpack"] = true;["_VERSION"] = true;["xpcall"] = true; - ["delay"] = true;["elapsedTime"] = true;["require"] = true;["spawn"] = true;["tick"] = true; - ["time"] = true;["typeof"] = true;["UserSettings"] = true;["wait"] = true;["warn"] = true; - ["game"] = true;["Enum"] = true;["script"] = true;["shared"] = true;["workspace"] = true; - ["Axes"] = true;["BrickColor"] = true;["CFrame"] = true;["Color3"] = true;["ColorSequence"] = true; - ["ColorSequenceKeypoint"] = true;["Faces"] = true;["Instance"] = true;["NumberRange"] = true; - ["NumberSequence"] = true;["NumberSequenceKeypoint"] = true;["PhysicalProperties"] = true; - ["Random"] = true;["Ray"] = true;["Rect"] = true;["Region3"] = true;["Region3int16"] = true; - ["TweenInfo"] = true;["UDim"] = true;["UDim2"] = true;["Vector2"] = true;["Vector3"] = true; - ["Vector3int16"] = true;["next"] = true; - ["os"] = true; - --["os.time"] = true;["os.date"] = true;["os.difftime"] = true; - ["debug"] = true; - --["debug.traceback"] = true;["debug.profilebegin"] = true;["debug.profileend"] = true; - ["math"] = true; - --["math.abs"] = true;["math.acos"] = true;["math.asin"] = true;["math.atan"] = true;["math.atan2"] = true;["math.ceil"] = true;["math.clamp"] = true;["math.cos"] = true;["math.cosh"] = true;["math.deg"] = true;["math.exp"] = true;["math.floor"] = true;["math.fmod"] = true;["math.frexp"] = true;["math.ldexp"] = true;["math.log"] = true;["math.log10"] = true;["math.max"] = true;["math.min"] = true;["math.modf"] = true;["math.noise"] = true;["math.pow"] = true;["math.rad"] = true;["math.random"] = true;["math.randomseed"] = true;["math.sign"] = true;["math.sin"] = true;["math.sinh"] = true;["math.sqrt"] = true;["math.tan"] = true;["math.tanh"] = true; - ["coroutine"] = true; - --["coroutine.create"] = true;["coroutine.resume"] = true;["coroutine.running"] = true;["coroutine.status"] = true;["coroutine.wrap"] = true;["coroutine.yield"] = true; - ["string"] = true; - --["string.byte"] = true;["string.char"] = true;["string.dump"] = true;["string.find"] = true;["string.format"] = true;["string.len"] = true;["string.lower"] = true;["string.match"] = true;["string.rep"] = true;["string.reverse"] = true;["string.sub"] = true;["string.upper"] = true;["string.gmatch"] = true;["string.gsub"] = true; - ["table"] = true; - --["table.concat"] = true;["table.insert"] = true;["table.remove"] = true;["table.sort"] = true; - } - end - return lua_builtin -end - ---- create a Lua token iterator from a string or file-like object. --- Will return the token type and value. --- @string s the string --- @tab[opt] filter a table of token types to exclude, by default `{space=true,comments=true}` --- @tab[opt] options a table of options; by default, `{number=true,string=true}`, --- which means convert numbers and strip string quotes. -function lexer.lua(s, filter, options) - filter = (filter or {space = true, comments = true}) - lexer.get_keywords() - lexer.get_builtins() - if (not lua_matches) then - lua_matches = { - {WSPACE, wsdump}, - {NUMBER3, ndump}, - {IDEN, lua_vdump}, - {NUMBER4, ndump}, - {NUMBER5, ndump}, - {STRING1, sdump}, - {STRING2, sdump}, - {STRING3, sdump}, - {"^%-%-%[(=*)%[.-%]%1%]", cdump}, - {"^%-%-.-\n", cdump}, - {"^%[(=*)%[.-%]%1%]", sdump_l}, - {"^==", tdump}, - {"^~=", tdump}, - {"^<=", tdump}, - {"^>=", tdump}, - {"^%.%.%.", tdump}, - {"^%.%.", tdump}, - {"^.", tdump} - } - end - return lexer.scan(s, lua_matches, filter, options) -end - ---- get a list of parameters separated by a delimiter from a stream. --- @param tok the token stream --- @string[opt=')'] endtoken end of list. Can be '\n' --- @string[opt=','] delim separator --- @return a list of token lists. -function lexer.get_separated_list(tok, endtoken, delim) - endtoken = (endtoken or ")") - delim = (delim or ",") - local parm_values = {} - local level = 1 -- used to count ( and ) - local tl = {} - local function tappend(tl, t, val) - val = (val or t) - append(tl, {t, val}) - end - local is_end - if (endtoken == "\n") then - is_end = function(t,val) - return (t == "space" and val:find("\n")) - end - else - is_end = function (t) - return (t == endtoken) - end - end - local token, value - while (true) do - token, value = tok() - if (not token) then return nil, "EOS" end -- end of stream is an error! - if (is_end(token, value) and level == 1) then - append(parm_values, tl) - break - elseif (token == "(") then - level = (level + 1) - tappend(tl, "(") - elseif (token == ")") then - level = (level - 1) - if (level == 0) then -- finished with parm list - append(parm_values, tl) - break - else - tappend(tl, ")") - end - elseif (token == delim and level == 1) then - append(parm_values, tl) -- a new parm - tl = {} - else - tappend(tl, token, value) - end - end - return parm_values, {token, value} -end - ---- get the next non-space token from the stream. --- @param tok the token stream. -function lexer.skipws(tok) - local t,v = tok() - while (t == "space") do - t,v = tok() - end - return t,v -end - -local skipws = lexer.skipws - ---- get the next token, which must be of the expected type. --- Throws an error if this type does not match! --- @param tok the token stream --- @string expected_type the token type --- @bool no_skip_ws whether we should skip whitespace -function lexer.expecting(tok, expected_type, no_skip_ws) - assert_arg(1, tok, "function") - assert_arg(2, expected_type, "string") - local t,v - if (no_skip_ws) then - t,v = tok() - else - t,v = skipws(tok) - end - if (t ~= expected_type) then error("expecting " .. expected_type, 2) end - return v -end - -return lexer]]> - - - - + true 0 @@ -2200,7 +1723,7 @@ return lexer]]> 1 - + false @@ -2242,7 +1765,7 @@ return lexer]]> true 1 - + false @@ -2284,7 +1807,7 @@ return lexer]]> true 1 - + false @@ -2326,7 +1849,7 @@ return lexer]]> false 1 - + false @@ -2380,7 +1903,7 @@ return lexer]]> true 1 - + 16 1 @@ -2389,7 +1912,7 @@ return lexer]]> - + UIPadding @@ -2411,7 +1934,7 @@ return lexer]]> - + false @@ -2453,7 +1976,7 @@ return lexer]]> true 1 - + false @@ -2510,7 +2033,7 @@ return lexer]]> - + false @@ -2552,7 +2075,7 @@ return lexer]]> true 1 - + false @@ -2615,7 +2138,7 @@ return lexer]]> true 1 - + false @@ -2658,7 +2181,7 @@ return lexer]]> 1 - + false @@ -2700,7 +2223,7 @@ return lexer]]> true 2 - + 1 1 @@ -2714,7 +2237,7 @@ return lexer]]> 1 - + false @@ -2756,7 +2279,7 @@ return lexer]]> false 1 - + true @@ -2832,7 +2355,7 @@ return lexer]]> 1 - + false @@ -2888,7 +2411,7 @@ return lexer]]> - + false @@ -2930,7 +2453,7 @@ return lexer]]> false 1 - + true @@ -3006,7 +2529,7 @@ return lexer]]> 1 - + false @@ -3061,7 +2584,7 @@ return lexer]]> 1 - + false @@ -3105,7 +2628,7 @@ return lexer]]> - + false @@ -3147,7 +2670,7 @@ return lexer]]> false 1 - + true @@ -3223,7 +2746,7 @@ return lexer]]> 1 - + false @@ -3281,7 +2804,7 @@ return lexer]]> - + false @@ -3323,7 +2846,7 @@ return lexer]]> true 1 - + false @@ -3378,7 +2901,7 @@ return lexer]]> 2 - + UIPadding @@ -3400,7 +2923,7 @@ return lexer]]> - + true @@ -3459,7 +2982,7 @@ return lexer]]> 1 - + false @@ -3518,7 +3041,7 @@ return lexer]]> - + false @@ -3560,7 +3083,7 @@ return lexer]]> true 1 - + false @@ -3602,7 +3125,7 @@ return lexer]]> true 1 - + false @@ -3665,7 +3188,7 @@ return lexer]]> true 1 - + false @@ -3707,7 +3230,7 @@ return lexer]]> true 2 - + false @@ -3749,7 +3272,7 @@ return lexer]]> true 1 - + false @@ -3804,7 +3327,7 @@ return lexer]]> 1 - + false @@ -3846,7 +3369,7 @@ return lexer]]> true 1 - + false @@ -3909,7 +3432,7 @@ return lexer]]> true 1 - + UIPadding @@ -3931,7 +3454,7 @@ return lexer]]> - + true @@ -3992,7 +3515,7 @@ return lexer]]> - + true @@ -4053,7 +3576,7 @@ return lexer]]> - + false @@ -4095,7 +3618,7 @@ return lexer]]> true 1 - + false @@ -4150,7 +3673,7 @@ return lexer]]> 1 - + false @@ -4192,7 +3715,7 @@ return lexer]]> true 1 - + false @@ -4255,7 +3778,7 @@ return lexer]]> true 1 - + UIPadding @@ -4277,7 +3800,7 @@ return lexer]]> - + true @@ -4338,7 +3861,7 @@ return lexer]]> - + true @@ -4399,7 +3922,7 @@ return lexer]]> - + false @@ -4441,7 +3964,7 @@ return lexer]]> true 1 - + true @@ -4499,7 +4022,7 @@ return lexer]]> true 1 - + false @@ -4554,7 +4077,7 @@ return lexer]]> 1 - + false @@ -4596,7 +4119,7 @@ return lexer]]> true 1 - + true @@ -4658,7 +4181,7 @@ return lexer]]> - + false @@ -4700,7 +4223,7 @@ return lexer]]> true 1 - + false @@ -4755,7 +4278,7 @@ return lexer]]> 1 - + false @@ -4797,7 +4320,7 @@ return lexer]]> true 1 - + false @@ -4860,7 +4383,7 @@ return lexer]]> true 1 - + UIPadding @@ -4882,7 +4405,7 @@ return lexer]]> - + true @@ -4944,7 +4467,7 @@ SetNumber(x)]]> - + true @@ -5005,7 +4528,7 @@ SetNumber(x)]]> - + false @@ -5047,7 +4570,7 @@ SetNumber(x)]]> true 1 - + false @@ -5102,7 +4625,7 @@ SetNumber(x)]]> 1 - + false @@ -5144,7 +4667,7 @@ SetNumber(x)]]> true 1 - + false @@ -5207,7 +4730,7 @@ SetNumber(x)]]> true 1 - + UIPadding @@ -5229,7 +4752,7 @@ SetNumber(x)]]> - + true @@ -5291,7 +4814,7 @@ SetNumber(x)]]> - + true @@ -5352,7 +4875,7 @@ SetNumber(x)]]> - + false @@ -5394,7 +4917,7 @@ SetNumber(x)]]> true 1 - + false @@ -5449,7 +4972,7 @@ SetNumber(x)]]> 1 - + false @@ -5491,7 +5014,7 @@ SetNumber(x)]]> true 1 - + false @@ -5554,7 +5077,7 @@ SetNumber(x)]]> true 1 - + UIPadding @@ -5576,7 +5099,7 @@ SetNumber(x)]]> - + true @@ -5638,7 +5161,7 @@ SetNumber(x)]]> - + true @@ -5699,7 +5222,7 @@ SetNumber(x)]]> - + false @@ -5741,7 +5264,7 @@ SetNumber(x)]]> true 1 - + false @@ -5796,7 +5319,7 @@ SetNumber(x)]]> 1 - + false @@ -5838,7 +5361,7 @@ SetNumber(x)]]> true 1 - + true @@ -5898,7 +5421,7 @@ SetNumber(x)]]> 1 - + true @@ -5959,7 +5482,7 @@ SetNumber(x)]]> - + true @@ -6018,7 +5541,7 @@ SetNumber(x)]]> 1 - + 1 0 @@ -6032,7 +5555,7 @@ SetNumber(x)]]> 1 - + UIPadding @@ -6055,7 +5578,7 @@ SetNumber(x)]]> - + false @@ -6101,7 +5624,7 @@ SetNumber(x)]]> - + false @@ -6143,7 +5666,7 @@ SetNumber(x)]]> true 1 - + false @@ -6185,7 +5708,7 @@ SetNumber(x)]]> true 1 - + false @@ -6248,7 +5771,7 @@ SetNumber(x)]]> false 1 - + false @@ -6290,7 +5813,7 @@ SetNumber(x)]]> true 2 - + UIPadding @@ -6312,7 +5835,7 @@ SetNumber(x)]]> - + false @@ -6356,7 +5879,7 @@ SetNumber(x)]]> - + false @@ -6400,7 +5923,7 @@ SetNumber(x)]]> - + false @@ -6442,7 +5965,7 @@ SetNumber(x)]]> true 1 - + UIPadding @@ -6464,7 +5987,7 @@ SetNumber(x)]]> - + false @@ -6523,6 +6046,269 @@ SetNumber(x)]]> + + + + Lexer + {7D941203-FF5D-430D-A233-5689182E8BAC} + ----------------- Original Penlight lexer author + ryanjmulder ----------------- Penlight lexer contributer + mpeterv ----------------- Penlight lexer contributer + Tieske ----------------- Penlight lexer contributer + boatbomber ----------------- Roblox port, optimizations, and bug fixes + Sleitnick ----------------- Roblox optimizations + + Usage: + + local source = "for i = 1,n do end" + + -- The 'scan' function returns a token iterator: + for token,src in lexer.scan(source) do + print(token, src) + end + + > keyword for + > iden i + > = = + > number 1 + > , , + > iden n + > keyword do + > keyword end + + List of tokens: + - keyword + - builtin + - iden + - string + - number + - space + - comment + + Other tokens that don't fall into the above categories + will simply be returned as itself. For instance, operators + like "+" will simply return "+" as the token. + +--]] + +local lexer = {} + +local yield, wrap = coroutine.yield, coroutine.wrap +local strfind = string.find +local strsub = string.sub +local append = table.insert +local type = type + +local NUMBER1 = "^[%+%-]?%d+%.?%d*[eE][%+%-]?%d+" +local NUMBER2 = "^[%+%-]?%d+%.?%d*" +local NUMBER3 = "^0x[%da-fA-F]+" +local NUMBER4 = "^%d+%.?%d*[eE][%+%-]?%d+" +local NUMBER5 = "^%d+%.?%d*" +local IDEN = "^[%a_][%w_]*" +local WSPACE = "^%s+" +local STRING1 = "^(['\"])%1" --Empty String +local STRING2 = [[^(['"])(\*)%2%1]] +local STRING3 = [[^(['"]).-[^\](\*)%2%1]] +local STRING4 = "^(['\"]).-.*" --Incompleted String +local STRING5 = "^%[(=*)%[.-%]%1%]" --Multiline-String +local STRING6 = "^%[%[.-.*" --Incompleted Multiline-String +local CHAR1 = "^''" +local CHAR2 = [[^'(\*)%1']] +local CHAR3 = [[^'.-[^\](\*)%1']] +local PREPRO = "^#.-[^\\]\n" +local MCOMMENT1 = "^%-%-%[(=*)%[.-%]%1%]" --Completed Multiline-Comment +local MCOMMENT2 = "^%-%-%[%[.-.*" --Incompleted Multiline-Comment +local SCOMMENT1 = "^%-%-.-\n" --Completed Singleline-Comment +local SCOMMENT2 = "^%-%-.-.*" --Incompleted Singleline-Comment + +local lua_keyword = { + ["and"] = true, ["break"] = true, ["do"] = true, ["else"] = true, ["elseif"] = true, + ["end"] = true, ["false"] = true, ["for"] = true, ["function"] = true, ["if"] = true, + ["in"] = true, ["local"] = true, ["nil"] = true, ["not"] = true, ["while"] = true, + ["or"] = true, ["repeat"] = true, ["return"] = true, ["then"] = true, ["true"] = true, + ["self"] = true, ["until"] = true +} + +local lua_builtin = { + ["assert"] = true;["collectgarbage"] = true;["error"] = true;["_G"] = true; + ["gcinfo"] = true;["getfenv"] = true;["getmetatable"] = true;["ipairs"] = true; + ["loadstring"] = true;["newproxy"] = true;["next"] = true;["pairs"] = true; + ["pcall"] = true;["print"] = true;["rawequal"] = true;["rawget"] = true;["rawset"] = true; + ["select"] = true;["setfenv"] = true;["setmetatable"] = true;["tonumber"] = true; + ["tostring"] = true;["type"] = true;["unpack"] = true;["_VERSION"] = true;["xpcall"] = true; + ["delay"] = true;["elapsedTime"] = true;["require"] = true;["spawn"] = true;["tick"] = true; + ["time"] = true;["typeof"] = true;["UserSettings"] = true;["wait"] = true;["warn"] = true; + ["game"] = true;["Enum"] = true;["script"] = true;["shared"] = true;["workspace"] = true; + ["Axes"] = true;["BrickColor"] = true;["CFrame"] = true;["Color3"] = true;["ColorSequence"] = true; + ["ColorSequenceKeypoint"] = true;["Faces"] = true;["Instance"] = true;["NumberRange"] = true; + ["NumberSequence"] = true;["NumberSequenceKeypoint"] = true;["PhysicalProperties"] = true; + ["Random"] = true;["Ray"] = true;["Rect"] = true;["Region3"] = true;["Region3int16"] = true; + ["TweenInfo"] = true;["UDim"] = true;["UDim2"] = true;["Vector2"] = true;["Vector3"] = true; + ["Vector3int16"] = true;["next"] = true; + ["os"] = true; + --["os.time"] = true;["os.date"] = true;["os.difftime"] = true; + ["debug"] = true; + --["debug.traceback"] = true;["debug.profilebegin"] = true;["debug.profileend"] = true; + ["math"] = true; + --["math.abs"] = true;["math.acos"] = true;["math.asin"] = true;["math.atan"] = true;["math.atan2"] = true;["math.ceil"] = true;["math.clamp"] = true;["math.cos"] = true;["math.cosh"] = true;["math.deg"] = true;["math.exp"] = true;["math.floor"] = true;["math.fmod"] = true;["math.frexp"] = true;["math.ldexp"] = true;["math.log"] = true;["math.log10"] = true;["math.max"] = true;["math.min"] = true;["math.modf"] = true;["math.noise"] = true;["math.pow"] = true;["math.rad"] = true;["math.random"] = true;["math.randomseed"] = true;["math.sign"] = true;["math.sin"] = true;["math.sinh"] = true;["math.sqrt"] = true;["math.tan"] = true;["math.tanh"] = true; + ["coroutine"] = true; + --["coroutine.create"] = true;["coroutine.resume"] = true;["coroutine.running"] = true;["coroutine.status"] = true;["coroutine.wrap"] = true;["coroutine.yield"] = true; + ["string"] = true; + --["string.byte"] = true;["string.char"] = true;["string.dump"] = true;["string.find"] = true;["string.format"] = true;["string.len"] = true;["string.lower"] = true;["string.match"] = true;["string.rep"] = true;["string.reverse"] = true;["string.sub"] = true;["string.upper"] = true;["string.gmatch"] = true;["string.gsub"] = true; + ["table"] = true; + --["table.concat"] = true;["table.insert"] = true;["table.remove"] = true;["table.sort"] = true; +} + +local function tdump(tok) + return yield(tok, tok) +end + +local function ndump(tok) + return yield("number", tok) +end + +local function sdump(tok) + return yield("string", tok) +end + +local function cdump(tok) + return yield("comment", tok) +end + +local function wsdump(tok) + return yield("space", tok) +end + +local function lua_vdump(tok) + if (lua_keyword[tok]) then + return yield("keyword", tok) + elseif (lua_builtin[tok]) then + return yield("builtin", tok) + else + return yield("iden", tok) + end +end + +local lua_matches = { + {IDEN, lua_vdump}, -- Indentifiers + {WSPACE, wsdump}, -- Whitespace + {NUMBER3, ndump}, -- Numbers + {NUMBER4, ndump}, + {NUMBER5, ndump}, + {STRING1, sdump}, -- Strings + {STRING2, sdump}, + {STRING3, sdump}, + {STRING4, sdump}, + {STRING5, sdump}, -- Multiline-Strings + {STRING6, sdump}, -- Multiline-Strings + + {MCOMMENT1, cdump}, -- Multiline-Comments + {MCOMMENT2, cdump}, + {SCOMMENT1, cdump}, -- Singleline-Comments + {SCOMMENT2, cdump}, + + {"^==", tdump}, -- Operators + {"^~=", tdump}, + {"^<=", tdump}, + {"^>=", tdump}, + {"^%.%.%.", tdump}, + {"^%.%.", tdump}, + {"^.", tdump} +} + +local num_lua_matches = #lua_matches + + +--- Create a plain token iterator from a string. +-- @tparam string s a string. +function lexer.scan(s) + + local function lex(first_arg) + + local line_nr = 0 + local sz = #s + local idx = 1 + + -- res is the value used to resume the coroutine. + local function handle_requests(res) + while (res) do + local tp = type(res) + -- Insert a token list: + if (tp == "table") then + res = yield("", "") + for i = 1,#res do + local t = res[i] + res = yield(t[1], t[2]) + end + elseif (tp == "string") then -- Or search up to some special pattern: + local i1, i2 = strfind(s, res, idx) + if (i1) then + local tok = strsub(s, i1, i2) + idx = (i2 + 1) + res = yield("", tok) + else + res = yield("", "") + idx = (sz + 1) + end + else + res = yield(line_nr, idx) + end + end + end + + handle_requests(first_arg) + line_nr = 1 + + while (true) do + + if (idx > sz) then + while (true) do + handle_requests(yield()) + end + end + + for i = 1,num_lua_matches do + local m = lua_matches[i] + local pat = m[1] + local fun = m[2] + local findres = {strfind(s, pat, idx)} + local i1, i2 = findres[1], findres[2] + if (i1) then + local tok = strsub(s, i1, i2) + idx = (i2 + 1) + lexer.finished = (idx > sz) + local res = fun(tok, findres) + if (tok:find("\n")) then + -- Update line number: + local _,newlines = tok:gsub("\n", {}) + line_nr = (line_nr + newlines) + end + handle_requests(res) + break + end + end + + end + + end + + return wrap(lex) + +end + +return lexer]]> + + + \ No newline at end of file