Calling exec() repeatedly with the same code string

[benjie-git]

Hi brythoners,

I have a brython web app in which the user can enter their own python code that in some cases can be run many times per second. I'm doing this using exec(), but as best I can tell, there is no caching of the parsed/transpiled js code when run repeatedly on the same string. Is there currently a way for me to cache the generated js code, and use that next time instead of re-generating it?

Instead of this:

clientVars = {}

def runCode(code):
    exec(code, clientVars)

runCode(userCode)

maybe something like this:

clientVars = {}
codeCache = {}

def runCode(pyCode):
    if pyCode not in codeCache:
        jsCode = py2js(pyCode)
        codeCache[pyCode] = jsCode
    else:
        jsCode = codeCache[pyCode]

    exec(jsCode, clientVars, precompiled=True)

runCode(userCode)

Thanks!

[denis-migdal]

Is there currently a way for me to cache the generated js code, and use that next time instead of re-generating it?

Yes, this is possible.

You can convert your code in JS, then execute it wherever you want :

__BRYTHON__.py2js(pycode, "exec").to_js();

See also: https://github.com/denis-migdal/Brython-AOT

[benjie-git]

Thanks Denis! But I also need to run my code using my brython app's clientVars context, so that if the user code creates variables and functions, they are available the next time it runs. Do you have any suggestions for doing that?

I'm now using code like this:

clientVars = {}
compiledCache = {}

def runCode(pyCode):
    if pyCode not in compiledCache:
        comp = compile(pyCode, "<string>", "exec")
        compiledCache[pyCode] = comp
    else:
        comp = compiledCache[pyCode]
    exec(comp, clientVars)


runCode("x = 0")
runCode("def addone(n): return n+1")

py = "x = addone(x); print(x)"
for _ in range(100):
    runCode(py)

which seems to work fine, but doesn't give me a measurable speed improvement.
This caches the string->ast step, but not the ast->js step. I'm hoping to be able to cache both.

[denis-migdal]

You can store the generated JS in the localStorage or in indexDB depending on the length of your scripts so that you can load them when the page refreshes.

If you have a lot of small scripts, I guess the best would be to merge them.

[benjie-git]

Thanks, but the different scripts will run at different times, and are editable by a user within the web app, and may change between runs, so I can't combine them into one, and cacheing between runs is not my issue. I'm just trying to make the scripts that run often (30x/sec) avoid recompilation into js every time, while still using a python dict as the globals context. Do you have a suggestion for how to run the compiled js code, using a python dict (clientVars) as the globals context, as my python exec() call does now?

Thanks!

[denis-migdal]

?

If you want to give him context, you can simply add an import at the start of your python script.

Else, I think there are stuff in the Brython doc.

[benjie-git]

To do some testing, I edited py_builtin_functions.js and added two new functions: compile2js() and execjs(). These variants of the originals return the python code compiled into js, instead of into ast form, and then run the js code with the given global and local context dicts. It works, but ends up not giving a huge efficiency savings over using the standard compile/exec calls, at least for my use case. So I think I'll give up on this for now, unless someone expresses interest, or has suggestions for speeding up my quick hacks.

Here are my modified js versions of compile() and exec(), for reference:

_b_.compile2js = function() {
    var $ = $B.args('compile2js', 5,
        {source:null, flags:null, dont_inherit:null,
         optimize:null, _feature_version:null},
         ['source', 'flags', 'dont_inherit', 'optimize',
             '_feature_version'],
         arguments,
         {flags: 0, dont_inherit: false, optimize: -1, _feature_version: 0},
         null, null)

    var module_name = '$exec_' + $B.UUID()
    $.__class__ = code
    $.co_flags = $.flags
    $.co_name = "<module>"
    $.filename = "<string>"
    $.mode = "exec"
    var filename = $.co_filename = $.filename
    var interactive = $.mode == "single" && ($.flags & 0x200)
    $B.file_cache[filename] = $.source
    $B.url2name[filename] = module_name

    if ($.flags & $B.PyCF_TYPE_COMMENTS) {
        // throw _b_.NotImplementedError.$factory('Brython does not currently support parsing of type comments')
    }

    if($B.$isinstance($.source, _b_.bytes)){
        var encoding = 'utf-8',
            lfpos = $.source.source.indexOf(10),
            first_line,
            second_line
        if(lfpos == -1){
            first_line = $.source
        }else{
            first_line = _b_.bytes.$factory($.source.source.slice(0, lfpos))
        }
        // decode with a safe decoder
        first_line = _b_.bytes.decode(first_line, 'latin-1')
        // search encoding (PEP263)
        var encoding_re = /^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)/
        var mo = first_line.match(encoding_re)
        if(mo){
            encoding = mo[1]
        }else if(lfpos > -1){
            // try second line
            var rest = $.source.source.slice(lfpos + 1)
            lfpos = rest.indexOf(10)
            if(lfpos > -1){
                second_line = _b_.bytes.$factory(rest.slice(0, lfpos))
            }else{
                second_line = _b_.bytes.$factory(rest)
            }
            second_line = _b_.bytes.decode(second_line, 'latin-1')
            mo = second_line.match(encoding_re)
            if(mo){
                encoding = mo[1]
            }
        }
        $.source = _b_.bytes.decode($.source, encoding)
    }

    if(! $B.$isinstance(filename, [_b_.bytes, _b_.str])){
        // module _warning is in builtin_modules.js
        $B.warn(_b_.DeprecationWarning,
            `path should be string, bytes, or os.PathLike, ` +
            `not ${$B.class_name(filename)}`)
    }
    if(interactive && ! $.source.endsWith("\n")){
        // This is used in codeop.py to raise SyntaxError until a block in the
        // interactive interpreter ends with "\n"
        // Cf. issue #853
        var lines = $.source.split("\n"),
            last_line = $B.last(lines)
        if(last_line.startsWith(" ")){
            var msg = "unexpected EOF while parsing",
                exc = _b_.SyntaxError.$factory()
            exc.filename = filename
            exc.lineno = exc.end_lineno = lines.length - 1
            exc.offset = 0
            exc.end_offset = last_line.length - 1
            exc.text = last_line
            exc.args = [msg, $B.fast_tuple([filename, exc.lineno, exc.offset,
                        exc.text, exc.end_lineno, exc.end_offset])]
            throw exc
        }
    }

    if($.source.__class__ && $.source.__class__.__module__ == 'ast'){
        // compile an ast instance
        $B.imported._ast._validate($.source)
        $._ast = $.source
        delete $.source
        return $
    }

    var _ast,
        parser


    // generated PEG parser
    try{
        var parser_mode = $.mode == 'eval' ? 'eval' : 'file'
        parser = new $B.Parser($.source, filename, parser_mode)
        parser.flags = $.flags
        _ast = $B._PyPegen.run_parser(parser)
    }catch(err){
        if($.mode == 'single'){
            var tester = parser.tokens[parser.tokens.length - 2]

            if(tester && (
                    (tester.type == "NEWLINE" && ($.flags & 0x4000)) ||
                    (tester.type == "DEDENT" && ($.flags & 0x200)))){
                err.__class__ = _b_.SyntaxError
                err.args[0] = 'incomplete input'
            }
        }
        throw err
    }
    if($.mode == 'single' && _ast.body.length == 1 &&
            _ast.body[0] instanceof $B.ast.Expr){
        // If mode is 'single' and the source is a single expression,
        // set _ast to an Expression and set attribute .single_expression
        // to compile() result. This is used in exec() to print the
        // expression if it is not None
        parser = new $B.Parser($.source, filename, 'eval')
        _ast = $B._PyPegen.run_parser(parser)
        $.single_expression = true
    }

    delete $B.url2name[filename]
    // Set attribute ._ast to avoid compiling again if result is passed to
    // exec()
    $._ast = $B.ast_js_to_py(_ast)
    $._ast.$js_ast = _ast

    // Compile the ast to JS, as in py2js, so we emit syntax errors created
    // by the JS conversion process.
    var future = $B.future_features(_ast, filename)
    var symtable = $B._PySymtable_Build(_ast, filename, future)
    var jsout = $B.js_from_root({
        ast: _ast,
        symtable,
        filename,
        src: $.source
    })

    return jsout.js
}


var $$execjs = _b_.execjs = function(){
    var $ = $B.args("execjs", 4,
            {js: null, globals: null, locals: null, mode: null} ,
            ['js', 'globals', 'locals', 'mode'],
            arguments, {globals: _b_.None, locals: _b_.None, mode: 'exec'},
            null, null, 4),
        js = $.js,
        _globals = $.globals,
        _locals = $.locals,
        mode = $.mode

    var filename = '<string>'

    var __name__ = '__main__'
    if(_globals !== _b_.None && _globals.__class__ == _b_.dict &&
            _b_.dict.$contains_string(_globals, '__name__')){
        __name__ = _b_.dict.$getitem_string(_globals, '__name__')
    }
    $B.url2name[filename] = __name__

    var frame = $B.frame_obj.frame

    $B.exec_scope = $B.exec_scope || {}

    var local_name = ('locals_' + __name__).replace(/\./g, '_'),
        global_name = ('globals_' + __name__).replace(/\./g, '_'),
        exec_locals = {},
        exec_globals = {}

    if(_globals === _b_.None){
        // if the optional parts are omitted, the code is executed in the
        // current scope
        // filename = '<string>'
        if(frame[1] === frame[3]){
            // module level
            global_name += '_globals'
            exec_locals = exec_globals = frame[3]
        }else{
            if(mode == "exec"){
                // for exec() : if the optional parts are omitted, the code is
                // executed in the current scope
                // modifications to the default locals dictionary should not
                // be attempted: this is why exec_locals is a clone of current
                // locals
                exec_locals = $B.clone(frame[1])
                for(var attr in frame[3]){
                    exec_locals[attr] = frame[3][attr]
                }
                exec_globals = exec_locals
            }else{
                // for eval() : If both dictionaries are omitted, the
                // expression is executed with the globals and locals in the
                // environment where eval() is called
                exec_locals = frame[1]
                exec_globals = frame[3]
            }
        }
    }else{
        if(_globals.__class__ !== _b_.dict){
            throw _b_.TypeError.$factory(`${mode}() globals must be ` +
                "a dict, not " + $B.class_name(_globals))
        }
        // _globals is used for both globals and locals
        exec_globals = {}
        if(_globals.$jsobj){ // eg globals()
            exec_globals = _globals.$jsobj
        }else{
            // The globals object must be the same across calls to exec()
            // with the same dictionary (cf. issue 690)
            exec_globals = _globals.$jsobj = {}
            for(var key of _b_.dict.$keys_string(_globals)){
                _globals.$jsobj[key] = _b_.dict.$getitem_string(_globals, key)
                if(key == '__name__'){
                    __name__ = _globals.$jsobj[key]
                }
            }
            _globals.$all_str = false
        }

        if(exec_globals.__builtins__ === undefined){
            exec_globals.__builtins__ = _b_.__builtins__
        }
        // filename = exec_globals.__file__ || '<string>'
        if(_locals === _b_.None){
            exec_locals = exec_globals
        }else{
            if(_locals === _globals){
                // running exec at module level
                global_name += '_globals'
                exec_locals = exec_globals
            }else if(_locals.$jsobj){
                for(let key in _locals.$jsobj){
                    exec_globals[key] = _locals.$jsobj[key]
                }
            }else{
                if(_locals.$jsobj){
                    exec_locals = _locals.$jsobj
                }else{
                    var klass = $B.get_class(_locals),
                        getitem = $B.$call($B.$getattr(klass, '__getitem__')),
                        setitem = $B.$call($B.$getattr(klass, '__setitem__'))
                    exec_locals = new Proxy(_locals, {
                        get(target, prop){
                            if(prop == '$target'){
                                return target
                            }
                            try{
                                return getitem(target, prop)
                            }catch(err){
                                return undefined
                            }
                        },
                        set(target, prop, value){
                            return setitem(target, prop, value)
                        }
                    })
                }
            }
        }
    }

    var save_frame_obj = $B.frame_obj
    var _ast

    frame = [__name__, exec_locals, __name__, exec_globals]
    frame.is_exec_top = true
    frame.__file__ = filename
    frame.$f_trace = $B.enter_frame(frame)
    var _frame_obj = $B.frame_obj
    frame.$lineno = 1

    try{
        var exec_func = new Function('$B', '_b_',
                                     local_name, global_name,
                                     'frame', '_frame_obj', js)
    }catch(err){
        if($B.get_option('debug') > 1){
            console.log('execjs() error\n', $B.format_indent(js, 0))
            console.log('-- js source\n', js)
        }
        $B.frame_obj = save_frame_obj
        throw err
    }

    // console.log('exec func', $B.format_indent(exec_func + '', 0))
    try{
        let old_locals = $B.imported["exec"]
        $B.imported["exec"] = exec_locals
        var res = exec_func($B, _b_, exec_locals, exec_globals, frame, _frame_obj)
        $B.imported["exec"] = old_locals
    }catch(err){
        if($B.get_option('debug') > 2){
            console.log(
                'JS code\n', js,
                '\nexecjs func', $B.format_indent(exec_func + '', 0),
                '\n    filename', filename,
                '\n    name from filename', $B.url2name[filename],
                '\n    local_name', local_name,
                '\n    exec_locals', exec_locals,
                '\n    global_name', global_name,
                '\n    exec_globals', exec_globals,
                '\n    frame', frame,
                '\n    _ast', _ast,
                '\n    js', js)
        }
        $B.frame_obj = save_frame_obj
        throw err
    }
    if(_globals !== _b_.None && ! _globals.$jsobj){
        for(var _key in exec_globals){
            if(! _key.startsWith('$')){
                _b_.dict.$setitem(_globals, _key, exec_globals[_key])
            }
        }
    }
    $B.frame_obj = save_frame_obj
}

$$execjs.$is_func = true