Skip to content

Commit

Permalink
morph: initial 3.11 support (no exception blocks yet), LOAD_GLOBAL fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@pulkin
pulkin committed Feb 5, 2024
1 parent a2a172b commit 58661da
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 42 deletions.
8 changes: 7 additions & 1 deletion pyteleport/bytecode/opcodes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
"""
python_feature_jump_2x = _python_version >= 0x030A
"""
Python 3.10 introduces a GEN_START no-op instruction. Python 3.11 and above re-works this further.
Python 3.10 introduces a GEN_START no-op instruction. Python 3.11 and above re-works this further towards RESUME.
"""
python_feature_gen_start_opcode = _python_version == 0x030A
"""
Expand All @@ -35,6 +35,12 @@
python_feature_pre_call = _python_version >= 0x030B
python_feature_cache = _python_version >= 0x030B
python_feature_load_global_null = _python_version >= 0x030B
python_feature_put_null = _python_version >= 0x030B
python_feature_resume_opcode = _python_version >= 0x030B
"""
Prior to Python 3.11 qualname is required for MAKE_FUNCTION
"""
python_feature_make_function_qualname = _python_version < 0x030B

# These unconditionally interrupt the normal bytecode flow
interrupting = tuple(
Expand Down
10 changes: 8 additions & 2 deletions pyteleport/bytecode/primitives.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,11 +271,17 @@ class NameInstruction2(NameInstruction):
The lowest bit (stands for loading NULL in LOAD_GLOBAL).
"""

def __str_arg__(self):
if self.bit:
return f"NULL + {self.arg}"
return str(self.arg)

def get_stack_effect(self, jump: bool = False) -> int:
return super().get_stack_effect(jump=jump) + self.bit
assert jump is False
return stack_effect(self.opcode, int(self.bit))

def encode(self, storage: NameStorage) -> EncodedInstruction:
return EncodedInstruction(self.opcode, storage.store(self.arg) << 1 + self.bit)
return EncodedInstruction(self.opcode, (storage.store(self.arg) << 1) + self.bit)


@dataclass(frozen=True)
Expand Down
106 changes: 67 additions & 39 deletions pyteleport/morph.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
from dataclasses import dataclass

from .bytecode import Bytecode, disassemble, jump_multiplier
from .bytecode.primitives import AbstractInstruction, NoArgInstruction, ConstInstruction, NameInstruction, EncodedInstruction, ReferencingInstruction, FloatingCell
from .bytecode.primitives import AbstractInstruction, NoArgInstruction, ConstInstruction, NameInstruction, \
NameInstruction2, EncodedInstruction, ReferencingInstruction, FloatingCell
from .bytecode.minias import assign_stack_size
from .primitives import NULL
from .bytecode.opcodes import (
Expand All @@ -24,25 +25,32 @@
IMPORT_NAME, IMPORT_FROM, MAKE_FUNCTION,
RAISE_VARARGS,
guess_entering_stack_size, python_feature_block_stack, python_feature_gen_start_opcode,
python_feature_resume_opcode, python_feature_load_global_null, python_feature_make_function_qualname,
python_feature_put_null
)
from .util import log_bytecode
from .storage import transmission_engine

EXCEPT_HANDLER = 257

# 3.9
code_object_args = ("argcount", "posonlyargcount", "kwonlyargcount",
"nlocals", "stacksize", "flags",
"code", "consts",
"names", "varnames",
"filename", "name", "firstlineno", "linetable",
"freevars", "cellvars",
)
# cpython/Lib/test/test_code.py
if python_feature_block_stack:
code_object_args = (
"argcount", "posonlyargcount", "kwonlyargcount", "nlocals", "stacksize", "flags", "code", "consts", "names",
"varnames", "filename", "name", "firstlineno", "linetable", "freevars", "cellvars",
) # no exceptiontable
else:
code_object_args = (
"argcount", "posonlyargcount", "kwonlyargcount", "nlocals", "stacksize", "flags", "code", "consts", "names",
"varnames", "filename", "name", "qualname", "firstlineno", "linetable", "exceptiontable", "freevars", "cellvars",
) # qualname as well

if python_feature_gen_start_opcode:
from .bytecode.opcodes import GEN_START
if python_feature_block_stack:
from .bytecode.opcodes import SETUP_FINALLY
if python_feature_load_global_null:
from .bytecode.opcodes import PUSH_NULL


def _iter_stack(value_stack, block_stack):
Expand All @@ -65,22 +73,23 @@ def _iter_stack(value_stack, block_stack):
"""
v_stack_iter = enumerate(value_stack, start=1)
cur_stack_level = 0
for block_stack_item in block_stack:
if python_feature_block_stack:
for block_stack_item in block_stack:

# check if items are coming in accending order
if block_stack_item.level < cur_stack_level:
raise ValueError(f"Illegal block_stack.level={block_stack_item.level}")
# check if items are coming in accending order
if block_stack_item.level < cur_stack_level:
raise ValueError(f"Illegal block_stack.level={block_stack_item.level}")

# output stack items up to the block stack level
while cur_stack_level < block_stack_item.level:
try:
cur_stack_level, stack_item = next(v_stack_iter)
except StopIteration:
raise StopIteration(f"Depleted value stack items ({cur_stack_level}) for block_stack.level={block_stack_item.level}")
yield stack_item, True
# output stack items up to the block stack level
while cur_stack_level < block_stack_item.level:
try:
cur_stack_level, stack_item = next(v_stack_iter)
except StopIteration:
raise StopIteration(f"Depleted value stack items ({cur_stack_level}) for block_stack.level={block_stack_item.level}")
yield stack_item, True

# output block stack item
yield block_stack_item, False
# output block stack item
yield block_stack_item, False

# output the rest of the value stack
for _, stack_item in v_stack_iter:
Expand Down Expand Up @@ -108,7 +117,10 @@ def from_bytecode(cls, code: Bytecode) -> "MorphCode":
return cls(code.instructions, current=code.current)

def get_marks(self):
return {self.instructions[self.editing]: "✎✎✎", **super().get_marks()}
result = super().get_marks()
if 0 <= self.editing < len(self.instructions):
result[self.instructions[self.editing]] = "✎✎✎"
return result

def insert_cell(self, cell: FloatingCell, at: Optional[int] = None):
if at is not None:
Expand All @@ -121,7 +133,7 @@ def insert_cell(self, cell: FloatingCell, at: Optional[int] = None):
def insert(self, instruction: AbstractInstruction, at: Optional[int] = None):
return self.insert_cell(FloatingCell(instruction), at=at)

def i(self, opcode: int, arg=NOTSET) -> FloatingCell:
def i(self, opcode: int, arg=NOTSET, bit: bool = False) -> FloatingCell:
if opcode < dis.HAVE_ARGUMENT:
if arg is not NOTSET:
raise ValueError(f"no argument expected for {dis.opname[opcode]}; provided: {arg=}")
Expand All @@ -134,7 +146,10 @@ def i(self, opcode: int, arg=NOTSET) -> FloatingCell:
elif opcode in dis.hasname + dis.haslocal + dis.hasfree:
if not isinstance(arg, str):
raise ValueError(f"string argument expected for {dis.opname[opcode]}; provided: {arg=}")
result = NameInstruction(opcode, arg)
if python_feature_load_global_null and opcode == LOAD_GLOBAL:
result = NameInstruction2(opcode, arg, bit)
else:
result = NameInstruction(opcode, arg)
elif opcode in dis.hasjabs + dis.hasjrel:
if not isinstance(arg, FloatingCell):
raise ValueError(f"cell argument expected for {dis.opname[opcode]}; provided: {arg=}")
Expand Down Expand Up @@ -171,12 +186,15 @@ def put_null(self) -> None:
"""
Puts a single NULL on the stack.
"""
# any unbound method will work here
# property.fget
# POP
self.i(LOAD_GLOBAL, "property")
self.i(LOAD_METHOD, "fget")
self.i(POP_TOP)
if python_feature_put_null:
self.i(PUSH_NULL)
else:
# any unbound method will work here
# property.fget
# POP
self.i(LOAD_GLOBAL, "property")
self.i(LOAD_METHOD, "fget")
self.i(POP_TOP)

def put_unpack(self, object_storage_name: str, object_storage: dict, tos) -> None:
"""
Expand Down Expand Up @@ -243,9 +261,11 @@ def unpack_storage(
Position in `code.co_consts` where the serialized data is
expected.
"""
# storage.loads(data) (kinda)
if python_feature_load_global_null:
self.put_null()
self.i(LOAD_CONST, object_storage_protocol.load_from_code.__code__)
self.i(LOAD_CONST, "unpack")
if python_feature_make_function_qualname:
self.i(LOAD_CONST, "unpack")
self.i(MAKE_FUNCTION, 0)
result = self.i(LOAD_CONST, object_data)
self.i(BUILD_TUPLE, 1)
Expand All @@ -265,9 +285,10 @@ def i_print(self, what: str):
what
The string to print.
"""
self.i(LOAD_GLOBAL, "print")
self.i(LOAD_CONST, what)
self.i(CALL_FUNCTION, 1)
self.i(LOAD_GLOBAL, "print", bit=1)
self.i(LOAD_CONST, (what,))
self.i(LOAD_CONST, {"flush": True})
self.i(CALL_FUNCTION_EX, 1)
self.i(POP_TOP)


Expand Down Expand Up @@ -317,8 +338,8 @@ def morph_into(p, nxt, call_nxt=False, object_storage=None, object_storage_name=
i.metadata.source.offset: i
for i in code.instructions
}
if python_feature_gen_start_opcode and code.instructions[0].instruction.opcode == GEN_START:
# Leave the generator header on top
if python_feature_resume_opcode or (python_feature_gen_start_opcode and code.instructions[0].instruction.opcode == GEN_START):
# Leave the header as-is
code.editing = 1
else:
code.editing = 0
Expand Down Expand Up @@ -390,7 +411,8 @@ def morph_into(p, nxt, call_nxt=False, object_storage=None, object_storage_name=
code.i(BUILD_TUPLE, 0)
code.i(CALL_FUNCTION_EX, 0)
elif isinstance(nxt, CodeType):
put(f"morph_into:{f_code.co_name}")
if python_feature_make_function_qualname:
put(f"morph_into:{f_code.co_name}")
code.i(MAKE_FUNCTION, 0)
code.i(BUILD_TUPLE, 0)
code.i(CALL_FUNCTION_EX, 0)
Expand Down Expand Up @@ -442,6 +464,12 @@ def morph_into(p, nxt, call_nxt=False, object_storage=None, object_storage_name=
linetable=f_code.co_lnotab,
exceptiontable=None,
)
if "qualname" in code_object_args:
init_args["qualname"] = f_code.co_qualname
if "exceptiontable" in code_object_args:
if f_code.co_exceptiontable:
raise ValueError(str(f_code.co_exceptiontable))
init_args["exceptiontable"] = f_code.co_exceptiontable
init_args = tuple(init_args[f"{i}"] for i in code_object_args)
result = CodeType(*init_args)

Expand Down

0 comments on commit 58661da

Please sign in to comment.