From 21f0432d4e90c3eebdbb18e5268002ff270bb40f Mon Sep 17 00:00:00 2001 From: kc611 Date: Sun, 26 Mar 2023 23:32:22 +0530 Subject: [PATCH 01/30] Changed basic datastructure interfaces --- README.md | 6 +- .../core/datastructures/basic_block.py | 98 ++--- numba_rvsdg/core/datastructures/block_map.py | 379 ------------------ numba_rvsdg/core/datastructures/byte_flow.py | 44 +- numba_rvsdg/core/datastructures/flow_info.py | 49 +-- numba_rvsdg/core/datastructures/labels.py | 64 ++- numba_rvsdg/core/datastructures/region.py | 24 ++ numba_rvsdg/core/datastructures/scfg.py | 328 +++++++++++++++ numba_rvsdg/core/transformations.py | 159 ++++++-- numba_rvsdg/rendering/rendering.py | 7 +- numba_rvsdg/tests/simulator.py | 4 +- numba_rvsdg/tests/test_block_map.py | 22 - numba_rvsdg/tests/test_blockmap.py | 104 ----- numba_rvsdg/tests/test_byteflow.py | 61 +-- numba_rvsdg/tests/test_scfg.py | 161 ++++++++ numba_rvsdg/tests/test_transforms.py | 98 ++--- 16 files changed, 849 insertions(+), 759 deletions(-) delete mode 100644 numba_rvsdg/core/datastructures/block_map.py create mode 100644 numba_rvsdg/core/datastructures/region.py create mode 100644 numba_rvsdg/core/datastructures/scfg.py delete mode 100644 numba_rvsdg/tests/test_block_map.py delete mode 100644 numba_rvsdg/tests/test_blockmap.py create mode 100644 numba_rvsdg/tests/test_scfg.py diff --git a/README.md b/README.md index 6386852..760f4bf 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ numba_rvsdg ├── core │   ├── datastructures │   │   ├── basic_block.py # BasicBlock implementation -│   │   ├── block_map.py # BlockMap implementation, maps labels to blocks -│   │   ├── byte_flow.py # ByteFlow implementation, BlockMap + bytecode +│   │   ├── scfg.py # SCFG implementation, maps labels to blocks +│   │   ├── byte_flow.py # ByteFlow implementation, SCFG + bytecode │   │   ├── flow_info.py # Converts program to ByteFlow │   │   └── labels.py # Collection of Label classes │   ├── transformations.py # Algorithms @@ -51,7 +51,7 @@ numba_rvsdg ├── networkx_vendored │   └── scc.py # Strongly Connected Componets (loop detection) ├── rendering -│   └── rendering.py # Graphivz based rendering of BlockMaps +│   └── rendering.py # Graphivz based rendering of SCFGs ├── tests │   ├── simulator.py # Simulator utility for running SCFGs │   ├── test_byteflow.py # Testung ByteFlow and others diff --git a/numba_rvsdg/core/datastructures/basic_block.py b/numba_rvsdg/core/datastructures/basic_block.py index 9a92375..3660fcc 100644 --- a/numba_rvsdg/core/datastructures/basic_block.py +++ b/numba_rvsdg/core/datastructures/basic_block.py @@ -1,47 +1,27 @@ import dis -from collections import ChainMap -from typing import Tuple, Dict, List -from dataclasses import dataclass, field, replace +from typing import Dict, List +from dataclasses import dataclass, field, InitVar -from numba_rvsdg.core.datastructures.labels import Label +from numba_rvsdg.core.datastructures.labels import Label, NameGenerator, BlockName from numba_rvsdg.core.utils import _next_inst_offset @dataclass(frozen=True) class BasicBlock: - label: Label - """The corresponding Label for this block. """ - - _jump_targets: Tuple[Label] = tuple() - """Jump targets (branch destinations) for this block""" - - backedges: Tuple[Label] = tuple() - """Backedges for this block.""" - - @property - def is_exiting(self) -> bool: - return not self.jump_targets - - @property - def fallthrough(self) -> bool: - return len(self._jump_targets) == 1 + name_gen: InitVar[NameGenerator] + """Block Name Generator associated with this BasicBlock. + Note: This is an initialization only argument and not + a class attribute.""" - @property - def jump_targets(self) -> Tuple[Label]: - acc = [] - for j in self._jump_targets: - if j not in self.backedges: - acc.append(j) - return tuple(acc) + block_name: BlockName = field(init=False) + """Unique name identifier for this block""" - def replace_backedge(self, target: Label) -> "BasicBlock": - if target in self.jump_targets: - assert not self.backedges - return replace(self, backedges=(target,)) - return self + label: Label + """The corresponding Label for this block.""" - def replace_jump_targets(self, jump_targets: Tuple) -> "BasicBlock": - return replace(self, _jump_targets=jump_targets) + def __post_init__(self, name_gen): + block_name = name_gen.new_block_name(label=self.label) + object.__setattr__(self, 'block_name', block_name) @dataclass(frozen=True) @@ -83,44 +63,18 @@ class BranchBlock(BasicBlock): variable: str = None branch_value_table: dict = None - def replace_jump_targets(self, jump_targets: Tuple) -> "BasicBlock": - fallthrough = len(jump_targets) == 1 - old_branch_value_table = self.branch_value_table - new_branch_value_table = {} - for target in self.jump_targets: - if target not in jump_targets: - # ASSUMPTION: only one jump_target is being updated - diff = set(jump_targets).difference(self.jump_targets) - assert len(diff) == 1 - new_target = next(iter(diff)) - for k, v in old_branch_value_table.items(): - if v == target: - new_branch_value_table[k] = new_target - else: - # copy all old values - for k, v in old_branch_value_table.items(): - if v == target: - new_branch_value_table[k] = v - - return replace( - self, - _jump_targets=jump_targets, - branch_value_table=new_branch_value_table, - ) +# Maybe we can register new blocks over here instead of static lists +block_types = { + 'basic': BasicBlock, + 'python_bytecode': PythonBytecodeBlock, + 'control_variable': ControlVariableBlock, + 'branch': BranchBlock +} -@dataclass(frozen=True) -class RegionBlock(BasicBlock): - kind: str = None - headers: Dict[Label, BasicBlock] = None - """The header of the region""" - subregion: "BlockMap" = None - """The subgraph excluding the headers - """ - exit: Label = None - """The exit node. - """ +def get_block_class(block_type_string): + if block_type_string in block_types: + return block_types[block_type_string] + else: + raise TypeError(f"Block Type {block_type_string} not recognized.") - def get_full_graph(self): - graph = ChainMap(self.subregion.graph, self.headers) - return graph diff --git a/numba_rvsdg/core/datastructures/block_map.py b/numba_rvsdg/core/datastructures/block_map.py deleted file mode 100644 index 26daf5c..0000000 --- a/numba_rvsdg/core/datastructures/block_map.py +++ /dev/null @@ -1,379 +0,0 @@ -import dis -import yaml -from textwrap import dedent -from typing import Set, Tuple, Dict, List, Iterator -from dataclasses import dataclass, field - -from numba_rvsdg.core.datastructures.basic_block import ( - BasicBlock, - ControlVariableBlock, - BranchBlock, - RegionBlock, -) -from numba_rvsdg.core.datastructures.labels import ( - Label, - ControlLabelGenerator, - SynthenticAssignment, - SyntheticExit, - SyntheticTail, - SyntheticReturn, - ControlLabel -) - - -@dataclass(frozen=True) -class BlockMap: - """Map of Labels to Blocks.""" - - graph: Dict[Label, BasicBlock] = field(default_factory=dict) - clg: ControlLabelGenerator = field( - default_factory=ControlLabelGenerator, compare=False - ) - - def __getitem__(self, index): - return self.graph[index] - - def __contains__(self, index): - return index in self.graph - - def __iter__(self): - """Graph Iterator""" - # initialise housekeeping datastructures - to_visit, seen = [self.find_head()], [] - while to_visit: - # get the next label on the list - label = to_visit.pop(0) - # if we have visited this, we skip it - if label in seen: - continue - else: - seen.append(label) - # get the corresponding block for the label - block = self[label] - # yield the label, block combo - yield (label, block) - # if this is a region, recursively yield everything from that region - if type(block) == RegionBlock: - for i in block.subregion: - yield i - # finally add any jump_targets to the list of labels to visit - to_visit.extend(block.jump_targets) - - def exclude_blocks(self, exclude_blocks: Set[Label]) -> Iterator[Label]: - """Iterator over all nodes not in exclude_blocks.""" - for block in self.graph: - if block not in exclude_blocks: - yield block - - def find_head(self) -> Label: - """Find the head block of the CFG. - - Assuming the CFG is closed, this will find the block - that no other blocks are pointing to. - - """ - heads = set(self.graph.keys()) - for label in self.graph.keys(): - block = self.graph[label] - for jt in block.jump_targets: - heads.discard(jt) - assert len(heads) == 1 - return next(iter(heads)) - - def compute_scc(self) -> List[Set[Label]]: - """ - Strongly-connected component for detecting loops. - """ - from numba_rvsdg.networkx_vendored.scc import scc - - class GraphWrap: - def __init__(self, graph): - self.graph = graph - - def __getitem__(self, vertex): - out = self.graph[vertex].jump_targets - # Exclude node outside of the subgraph - return [k for k in out if k in self.graph] - - def __iter__(self): - return iter(self.graph.keys()) - - return list(scc(GraphWrap(self.graph))) - - def compute_scc_subgraph(self, subgraph) -> List[Set[Label]]: - """ - Strongly-connected component for detecting loops inside a subgraph. - """ - from numba_rvsdg.networkx_vendored.scc import scc - - class GraphWrap: - def __init__(self, graph, subgraph): - self.graph = graph - self.subgraph = subgraph - - def __getitem__(self, vertex): - out = self.graph[vertex].jump_targets - # Exclude node outside of the subgraph - return [k for k in out if k in subgraph] - - def __iter__(self): - return iter(self.graph.keys()) - - return list(scc(GraphWrap(self.graph, subgraph))) - - def find_headers_and_entries( - self, subgraph: Set[Label] - ) -> Tuple[Set[Label], Set[Label]]: - """Find entries and headers in a given subgraph. - - Entries are blocks outside the subgraph that have an edge pointing to - the subgraph headers. Headers are blocks that are part of the strongly - connected subset and that have incoming edges from outside the - subgraph. Entries point to headers and headers are pointed to by - entries. - - """ - outside: Label - entries: Set[Label] = set() - headers: Set[Label] = set() - - for outside in self.exclude_blocks(subgraph): - nodes_jump_in_loop = subgraph.intersection(self.graph[outside].jump_targets) - headers.update(nodes_jump_in_loop) - if nodes_jump_in_loop: - entries.add(outside) - # If the loop has no headers or entries, the only header is the head of - # the CFG. - if not headers: - headers = {self.find_head()} - return headers, entries - - def find_exiting_and_exits( - self, subgraph: Set[Label] - ) -> Tuple[Set[Label], Set[Label]]: - """Find exiting and exit blocks in a given subgraph. - - Existing blocks are blocks inside the subgraph that have edges to - blocks outside of the subgraph. Exit blocks are blocks outside the - subgraph that have incoming edges from within the subgraph. Exiting - blocks point to exits and exits and pointed to by exiting blocks. - - """ - inside: Label - exiting: Set[Label] = set() - exits: Set[Label] = set() - for inside in subgraph: - # any node inside that points outside the loop - for jt in self.graph[inside].jump_targets: - if jt not in subgraph: - exiting.add(inside) - exits.add(jt) - # any returns - if self.graph[inside].is_exiting: - exiting.add(inside) - return exiting, exits - - def is_reachable_dfs(self, begin: Label, end: Label): # -> TypeGuard: - """Is end reachable from begin.""" - seen = set() - to_vist = list(self.graph[begin].jump_targets) - while True: - if to_vist: - block = to_vist.pop() - else: - return False - - if block in seen: - continue - elif block == end: - return True - elif block not in seen: - seen.add(block) - if block in self.graph: - to_vist.extend(self.graph[block].jump_targets) - - def add_block(self, basicblock: BasicBlock): - self.graph[basicblock.label] = basicblock - - def remove_blocks(self, labels: Set[Label]): - for label in labels: - del self.graph[label] - - def insert_block( - self, new_label: Label, predecessors: Set[Label], successors: Set[Label] - ): - # TODO: needs a diagram and documentaion - # initialize new block - new_block = BasicBlock( - label=new_label, _jump_targets=successors, backedges=set() - ) - # add block to self - self.add_block(new_block) - # Replace any arcs from any of predecessors to any of successors with - # an arc through the inserted block instead. - for label in predecessors: - block = self.graph.pop(label) - jt = list(block.jump_targets) - if successors: - for s in successors: - if s in jt: - if new_label not in jt: - jt[jt.index(s)] = new_label - else: - jt.pop(jt.index(s)) - else: - jt.append(new_label) - self.add_block(block.replace_jump_targets(jump_targets=tuple(jt))) - - def insert_block_and_control_blocks( - self, new_label: Label, predecessors: Set[Label], successors: Set[Label] - ): - # TODO: needs a diagram and documentaion - # name of the variable for this branching assignment - branch_variable = self.clg.new_variable() - # initial value of the assignment - branch_variable_value = 0 - # store for the mapping from variable value to label - branch_value_table = {} - # Replace any arcs from any of predecessors to any of successors with - # an arc through the to be inserted block instead. - for label in predecessors: - block = self.graph[label] - jt = list(block.jump_targets) - # Need to create synthetic assignments for each arc from a - # predecessors to a successor and insert it between the predecessor - # and the newly created block - for s in set(jt).intersection(successors): - synth_assign = SynthenticAssignment(str(self.clg.new_index())) - variable_assignment = {} - variable_assignment[branch_variable] = branch_variable_value - synth_assign_block = ControlVariableBlock( - label=synth_assign, - _jump_targets=(new_label,), - backedges=(), - variable_assignment=variable_assignment, - ) - # add block - self.add_block(synth_assign_block) - # update branching table - branch_value_table[branch_variable_value] = s - # update branching variable - branch_variable_value += 1 - # replace previous successor with synth_assign - jt[jt.index(s)] = synth_assign - # finally, replace the jump_targets - self.add_block( - self.graph.pop(label).replace_jump_targets(jump_targets=tuple(jt)) - ) - # initialize new block, which will hold the branching table - new_block = BranchBlock( - label=new_label, - _jump_targets=tuple(successors), - backedges=set(), - variable=branch_variable, - branch_value_table=branch_value_table, - ) - # add block to self - self.add_block(new_block) - - def join_returns(self): - """Close the CFG. - - A closed CFG is a CFG with a unique entry and exit node that have no - predescessors and no successors respectively. - """ - # for all nodes that contain a return - return_nodes = [node for node in self.graph if self.graph[node].is_exiting] - # close if more than one is found - if len(return_nodes) > 1: - return_solo_label = SyntheticReturn(str(self.clg.new_index())) - self.insert_block(return_solo_label, return_nodes, tuple()) - - def join_tails_and_exits(self, tails: Set[Label], exits: Set[Label]): - if len(tails) == 1 and len(exits) == 1: - # no-op - solo_tail_label = next(iter(tails)) - solo_exit_label = next(iter(exits)) - return solo_tail_label, solo_exit_label - - if len(tails) == 1 and len(exits) == 2: - # join only exits - solo_tail_label = next(iter(tails)) - solo_exit_label = SyntheticExit(str(self.clg.new_index())) - self.insert_block(solo_exit_label, tails, exits) - return solo_tail_label, solo_exit_label - - if len(tails) >= 2 and len(exits) == 1: - # join only tails - solo_tail_label = SyntheticTail(str(self.clg.new_index())) - solo_exit_label = next(iter(exits)) - self.insert_block(solo_tail_label, tails, exits) - return solo_tail_label, solo_exit_label - - if len(tails) >= 2 and len(exits) >= 2: - # join both tails and exits - solo_tail_label = SyntheticTail(str(self.clg.new_index())) - solo_exit_label = SyntheticExit(str(self.clg.new_index())) - self.insert_block(solo_tail_label, tails, exits) - self.insert_block(solo_exit_label, set((solo_tail_label,)), exits) - return solo_tail_label, solo_exit_label - - @staticmethod - def bcmap_from_bytecode(bc: dis.Bytecode): - return {inst.offset: inst for inst in bc} - - @staticmethod - def from_yaml(yaml_string): - data = yaml.safe_load(yaml_string) - return BlockMap.from_dict(data) - - @staticmethod - def from_dict(graph_dict): - block_map_graph = {} - clg = ControlLabelGenerator() - for index, attributes in graph_dict.items(): - jump_targets = attributes["jt"] - backedges = attributes.get("be", ()) - label = ControlLabel(str(clg.new_index())) - block = BasicBlock( - label=label, - backedges=wrap_id(backedges), - _jump_targets=wrap_id(jump_targets), - ) - block_map_graph[label] = block - return BlockMap(block_map_graph, clg=clg) - - def to_yaml(self): - # Convert to yaml - block_map_graph = self.graph - yaml_string = """""" - - for key, value in block_map_graph.items(): - jump_targets = [f"{i.index}" for i in value._jump_targets] - jump_targets = str(jump_targets).replace("\'", "\"") - back_edges = [f"{i.index}" for i in value.backedges] - jump_target_str= f""" - "{str(key.index)}": - jt: {jump_targets}""" - - if back_edges: - back_edges = str(back_edges).replace("\'", "\"") - jump_target_str += f""" - be: {back_edges}""" - yaml_string += dedent(jump_target_str) - - return yaml_string - - def to_dict(self): - block_map_graph = self.graph - graph_dict = {} - for key, value in block_map_graph.items(): - curr_dict = {} - curr_dict["jt"] = [f"{i.index}" for i in value._jump_targets] - if value.backedges: - curr_dict["be"] = [f"{i.index}" for i in value.backedges] - graph_dict[str(key.index)] = curr_dict - return graph_dict - -def wrap_id(indices: Set[Label]): - return tuple([ControlLabel(i) for i in indices]) diff --git a/numba_rvsdg/core/datastructures/byte_flow.py b/numba_rvsdg/core/datastructures/byte_flow.py index b77859c..3fe38a7 100644 --- a/numba_rvsdg/core/datastructures/byte_flow.py +++ b/numba_rvsdg/core/datastructures/byte_flow.py @@ -1,19 +1,17 @@ import dis -from copy import deepcopy from dataclasses import dataclass -from numba_rvsdg.core.datastructures.block_map import BlockMap -from numba_rvsdg.core.datastructures.basic_block import RegionBlock +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.flow_info import FlowInfo from numba_rvsdg.core.utils import _logger, _LogWrap -from numba_rvsdg.core.transformations import restructure_loop, restructure_branch +from numba_rvsdg.core.transformations import restructure_loop, restructure_branch, join_returns @dataclass(frozen=True) class ByteFlow: bc: dis.Bytecode - bbmap: "BlockMap" + bbmap: SCFG @staticmethod def from_bytecode(code) -> "ByteFlow": @@ -25,41 +23,23 @@ def from_bytecode(code) -> "ByteFlow": return ByteFlow(bc=bc, bbmap=bbmap) def _join_returns(self): - bbmap = deepcopy(self.bbmap) - bbmap.join_returns() - return ByteFlow(bc=self.bc, bbmap=bbmap) + join_returns(self.bbmap) def _restructure_loop(self): - bbmap = deepcopy(self.bbmap) - restructure_loop(bbmap) - for region in _iter_subregions(bbmap): - restructure_loop(region.subregion) - return ByteFlow(bc=self.bc, bbmap=bbmap) + restructure_loop(self.bbmap) def _restructure_branch(self): - bbmap = deepcopy(self.bbmap) - restructure_branch(bbmap) - for region in _iter_subregions(bbmap): - restructure_branch(region.subregion) - return ByteFlow(bc=self.bc, bbmap=bbmap) + restructure_branch(self.bbmap) def restructure(self): - bbmap = deepcopy(self.bbmap) # close - bbmap.join_returns() + join_returns(self.bbmap) # handle loop - restructure_loop(bbmap) - for region in _iter_subregions(bbmap): - restructure_loop(region.subregion) + restructure_loop(self.bbmap) # handle branch - restructure_branch(bbmap) - for region in _iter_subregions(bbmap): - restructure_branch(region.subregion) - return ByteFlow(bc=self.bc, bbmap=bbmap) + restructure_branch(self.bbmap) + @staticmethod + def bcmap_from_bytecode(bc: dis.Bytecode): + return {inst.offset: inst for inst in bc} -def _iter_subregions(bbmap: "BlockMap"): - for node in bbmap.graph.values(): - if isinstance(node, RegionBlock): - yield node - yield from _iter_subregions(node.subregion) diff --git a/numba_rvsdg/core/datastructures/flow_info.py b/numba_rvsdg/core/datastructures/flow_info.py index 3caa41b..3ead8d8 100644 --- a/numba_rvsdg/core/datastructures/flow_info.py +++ b/numba_rvsdg/core/datastructures/flow_info.py @@ -1,13 +1,11 @@ import dis - from typing import Set, Tuple, Dict, Sequence from dataclasses import dataclass, field from numba_rvsdg.core.datastructures.basic_block import PythonBytecodeBlock -from numba_rvsdg.core.datastructures.block_map import BlockMap +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.labels import ( - Label, - ControlLabelGenerator, + BlockName, PythonBytecodeLabel, ) from numba_rvsdg.core.utils import ( @@ -35,10 +33,6 @@ class FlowInfo: """Offset of the last bytecode instruction. """ - clg: ControlLabelGenerator = field( - default_factory=ControlLabelGenerator, compare=False - ) - def _add_jump_inst(self, offset: int, targets: Sequence[int]): """Add jump instruction to FlowInfo.""" for off in targets: @@ -71,33 +65,34 @@ def from_bytecode(bc: dis.Bytecode) -> "FlowInfo": flowinfo.last_offset = inst.offset return flowinfo - def build_basicblocks(self: "FlowInfo", end_offset=None) -> "BlockMap": + def build_basicblocks(self: "FlowInfo", end_offset=None) -> "SCFG": """ Build a graph of basic-blocks """ offsets = sorted(self.block_offsets) - # enumerate labels - labels = dict( - (offset, PythonBytecodeLabel(self.clg.new_index())) for offset in offsets - ) + scfg = SCFG() + + names = {} if end_offset is None: end_offset = _next_inst_offset(self.last_offset) - bbmap = BlockMap(graph={}, clg=self.clg) + for begin, end in zip(offsets, [*offsets[1:], end_offset]): - label = labels[begin] - targets: Tuple[Label, ...] + names[begin] = scfg.add_block(block_type=PythonBytecodeBlock, + begin=begin, + end=end, + label=PythonBytecodeLabel()) + + for begin, end in zip(offsets, [*offsets[1:], end_offset]): + targets: Tuple[BlockName, ...] term_offset = _prev_inst_offset(end) if term_offset not in self.jump_insts: # implicit jump - targets = (labels[end],) + targets = (names[end],) else: - targets = tuple(labels[o] for o in self.jump_insts[term_offset]) - block = PythonBytecodeBlock( - label=label, - begin=begin, - end=end, - _jump_targets=targets, - backedges=(), - ) - bbmap.add_block(block) - return bbmap + targets = tuple(names[o] for o in self.jump_insts[term_offset]) + + block_name = names[begin] + scfg.add_connections(block_name, targets, []) + + scfg.check_graph() + return scfg diff --git a/numba_rvsdg/core/datastructures/labels.py b/numba_rvsdg/core/datastructures/labels.py index 648f941..4db2e3f 100644 --- a/numba_rvsdg/core/datastructures/labels.py +++ b/numba_rvsdg/core/datastructures/labels.py @@ -1,9 +1,11 @@ from dataclasses import dataclass +from typing import List @dataclass(frozen=True, order=True) class Label: - index: int + info: List[str] = None + """Any Block specific information we want to add can go here""" ... @@ -57,17 +59,57 @@ class SynthenticAssignment(ControlLabel): pass -class ControlLabelGenerator: - def __init__(self, index=0, variable=97): - self.index = index - self.variable = variable +# Maybe we can register new labels over here instead of static lists +label_types = { + 'label': Label, + 'python_bytecode': PythonBytecodeLabel, + 'control': ControlLabel, + 'synth_branch': SyntheticBranch, + 'synth_tail': SyntheticTail, + 'synth_exit': SyntheticExit, + 'synth_head': SyntheticHead, + 'synth_return': SyntheticReturn, + 'synth_latch': SyntheticLatch, + 'synth_exit_latch': SyntheticExitingLatch, + 'synth_assign': SynthenticAssignment +} - def new_index(self): + +def get_label_class(label_type_string): + if label_type_string in label_types: + return label_types[label_type_string] + else: + raise TypeError(f"Block Type {label_type_string} not recognized.") + + + +@dataclass(frozen=True, order=True) +class BlockName: + name: str + ... + +@dataclass(frozen=True, order=True) +class RegionName: + name: str + ... + +@dataclass +class NameGenerator: + index: int = 0 + var_index: int = 0 + region_index: int = 0 + + def new_block_name(self, label): ret = self.index self.index += 1 - return ret + return BlockName(str(label).lower().split("(")[0] + "_" + str(ret)) + + def new_region_name(self, kind): + ret = self.region_index + self.region_index += 1 + return RegionName(str(kind).lower().split("(")[0] + "_" + str(ret)) - def new_variable(self): - ret = chr(self.variable) - self.variable += 1 - return ret + def new_var_name(self): + var_name = chr(self.var_index) + self.var_index = self.var_index + 1 + return str(var_name) diff --git a/numba_rvsdg/core/datastructures/region.py b/numba_rvsdg/core/datastructures/region.py new file mode 100644 index 0000000..147ed34 --- /dev/null +++ b/numba_rvsdg/core/datastructures/region.py @@ -0,0 +1,24 @@ + +from dataclasses import dataclass, field, InitVar + +from numba_rvsdg.core.datastructures.labels import NameGenerator, RegionName, BlockName + + +@dataclass(frozen=True) +class Region: + name_gen: InitVar[NameGenerator] + """Region Name Generator associated with this Region. + Note: This is an initialization only argument and not + a class attribute.""" + + region_name: RegionName = field(init=False) + """Unique name identifier for this region""" + + kind: str + header: BlockName + exiting: BlockName + + def __post_init__(self, name_gen): + region_name = name_gen.new_region_name(kind=self.kind) + object.__setattr__(self, 'region_name', region_name) + diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py new file mode 100644 index 0000000..21691d5 --- /dev/null +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -0,0 +1,328 @@ + +import yaml +import itertools + +from textwrap import dedent +from typing import Set, Tuple, Dict, List, Iterator +from dataclasses import dataclass, field + +from numba_rvsdg.core.datastructures.basic_block import ( + BasicBlock, + get_block_class +) +from numba_rvsdg.core.datastructures.region import Region +from numba_rvsdg.core.datastructures.labels import ( + BlockName, + NameGenerator, + RegionName, + get_label_class +) + + +@dataclass(frozen=True) +class SCFG: + """Maps of BlockNames to respective BasicBlocks. + And stores the jump targets and back edges for + blocks within the graph.""" + + blocks: Dict[BlockName, BasicBlock] = field(default_factory=dict) + + out_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) + back_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) + in_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) # Design question + + regions: Dict[RegionName, Region] = field(default_factory=dict) + + name_gen: NameGenerator = field( + default_factory=NameGenerator, compare=False + ) + + def __getitem__(self, index): + return self.blocks[index] + + def __contains__(self, index): + return index in self.blocks + + def __iter__(self): + """Graph Iterator""" + # initialise housekeeping datastructures + to_visit, seen = [self.find_head()], [] + while to_visit: + # get the next block_name on the list + block_name = to_visit.pop(0) + # if we have visited this, we skip it + if block_name in seen: + continue + else: + seen.append(block_name) + # get the corresponding block for the block_name + block = self[block_name] + # yield the block_name, block combo + yield (block_name, block) + # finally add any out_edges to the list of block_names to visit + to_visit.extend(self.out_edges[block_name]) + + def exclude_blocks(self, exclude_blocks: Set[BlockName]) -> Iterator[BlockName]: + """Iterator over all nodes not in exclude_blocks.""" + for block in self.blocks: + if block not in exclude_blocks: + yield block + + def find_head(self) -> BlockName: + """Find the head block of the CFG. + + Assuming the CFG is closed, this will find the block + that no other blocks are pointing to. + + """ + heads = set(self.blocks.keys()) + for name in self.blocks.keys(): + for jt in self.out_edges[name]: + heads.discard(jt) + assert len(heads) == 1 + return next(iter(heads)) + + def compute_scc(self) -> List[Set[BlockName]]: + """ + Strongly-connected component for detecting loops. + """ + from numba_rvsdg.networkx_vendored.scc import scc + + out_edges = self.out_edges + + class GraphWrap: + def __init__(self, graph): + self.graph = graph + + def __getitem__(self, vertex): + out = out_edges[self.graph[vertex]] + # Exclude node outside of the subgraph + return [k for k in out if k in self.graph] + + def __iter__(self): + return iter(self.graph.keys()) + + return list(scc(GraphWrap(self.blocks))) + + def compute_scc_subgraph(self, subgraph) -> List[Set[BlockName]]: + """ + Strongly-connected component for detecting loops inside a subgraph. + """ + from numba_rvsdg.networkx_vendored.scc import scc + out_edges = self.out_edges + + class GraphWrap: + def __init__(self, graph, subgraph): + self.graph = graph + self.subgraph = subgraph + + def __getitem__(self, vertex): + out = out_edges[self.graph[vertex]] + # Exclude node outside of the subgraph + return [k for k in out if k in subgraph] + + def __iter__(self): + return iter(self.graph.keys()) + + return list(scc(GraphWrap(self.blocks, subgraph))) + + def find_headers_and_entries( + self, subgraph: Set[BlockName] + ) -> Tuple[Set[BlockName], Set[BlockName]]: + """Find entries and headers in a given subgraph. + + Entries are blocks outside the subgraph that have an edge pointing to + the subgraph headers. Headers are blocks that are part of the strongly + connected subset and that have incoming edges from outside the + subgraph. Entries point to headers and headers are pointed to by + entries. + + """ + outside: BlockName + entries: Set[BlockName] = set() + headers: Set[BlockName] = set() + + for outside in self.exclude_blocks(subgraph): + nodes_jump_in_loop = subgraph.intersection(self.out_edges[self.blocks[outside]]) + headers.update(nodes_jump_in_loop) + if nodes_jump_in_loop: + entries.add(outside) + # If the loop has no headers or entries, the only header is the head of + # the CFG. + if not headers: + headers = {self.find_head()} + return headers, entries + + def find_exiting_and_exits( + self, subgraph: Set[BlockName] + ) -> Tuple[Set[BlockName], Set[BlockName]]: + """Find exiting and exit blocks in a given subgraph. + + Existing blocks are blocks inside the subgraph that have edges to + blocks outside of the subgraph. Exit blocks are blocks outside the + subgraph that have incoming edges from within the subgraph. Exiting + blocks point to exits and exits and pointed to by exiting blocks. + + """ + inside: BlockName + exiting: Set[BlockName] = set() + exits: Set[BlockName] = set() + for inside in subgraph: + # any node inside that points outside the loop + for jt in self.out_edges[self.blocks[inside]]: + if jt not in subgraph: + exiting.add(inside) + exits.add(jt) + # any returns + if self.blocks[inside].is_exiting: + exiting.add(inside) + return exiting, exits + + def is_reachable_dfs(self, begin: BlockName, end: BlockName): # -> TypeGuard: + """Is end reachable from begin.""" + seen = set() + to_vist = list(self.out_edges[self.blocks[begin]]) + while True: + if to_vist: + block = to_vist.pop() + else: + return False + + if block in seen: + continue + elif block == end: + return True + elif block not in seen: + seen.add(block) + if block in self.blocks: + to_vist.extend(self.out_edges[self.blocks[block]]) + + def check_graph(self): + pass + + # We don't need this cause everything is 'hopefully' additive + # def remove_blocks(self, names: Set[BlockName]): + # for name in names: + # del self.blocks[name] + # del self.out_edges[name] + # del self.back_edges[name] + # del self.in_edges[name] + # self.check_graph() + + def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], block_type: type = BasicBlock, **block_args): + # TODO: needs a diagram and documentaion + # initialize new block + + new_block_name = self.add_block(block_type, **block_args) + # Replace any arcs from any of predecessors to any of successors with + # an arc through the inserted block instead. + + for pred_name in predecessors: + # For every predecessor + # Remove all successors from its out edges + self.out_edges[pred_name].difference_update(successors) + # Add the inserted block as out edge + self.out_edges[pred_name].add(new_block_name) + # For inserted block, the predecessor in an in-edge + self.in_edges[new_block_name].add(pred_name) + + for success_name in successors: + # For every sucessor + # Remove all predecessors from it's in edges + self.in_edges[success_name].difference_update(predecessors) + # Add the inserted block as in edge + self.in_edges[success_name].add(new_block_name) + # For inserted block, the sucessor in an out-edge + self.out_edges[new_block_name].add(success_name) + + self.check_graph() + return new_block_name + + def add_block(self, block_type: str = 'basic', **block_args): + block_type = get_block_class(block_type) + new_block = block_type( + **block_args, + name_gen=self.name_gen + ) + + name = new_block.block_name + self.blocks[name] = new_block + + self.in_edges[name] = set() + self.back_edges[name] = set() + self.out_edges[name] = set() + + return name + + def add_connections(self, block_name, out_edges=(), back_edges=()): + self.out_edges[block_name] = out_edges + self.back_edges[block_name] = back_edges + + for edge in itertools.chain(out_edges, back_edges): + self.in_edges[edge].add(block_name) + + self.check_graph() + + + @staticmethod + def from_yaml(yaml_string): + data = yaml.safe_load(yaml_string) + return SCFG.from_dict(data) + + @staticmethod + def from_dict(graph_dict): + scfg = SCFG() + ref_dict = {} + + for block_ref, block_attrs in graph_dict.items(): + block_class = get_block_class(block_attrs['type']) + block_args = {} + label_class = get_label_class(block_attrs.get('label_type', 'label')) + label_info = block_attrs.get('label_info', None) + block_args = {'label': label_class(label_info)} + block_name = scfg.add_block(block_class, **block_args) + ref_dict[block_ref] = block_name + + for block_ref, block_attrs in graph_dict.items(): + out_refs = block_attrs.get("out", set()) + back_refs = block_attrs.get("back", set()) + + block_name = ref_dict[block_ref] + out_edges = set(ref_dict[out_ref] for out_ref in out_refs) + back_edges = set(ref_dict[back_ref] for back_ref in back_refs) + scfg.add_connections(block_name, out_edges, back_edges) + + scfg.check_graph() + return scfg, ref_dict + + def to_yaml(self): + # Convert to yaml + yaml_string = """""" + + for key, value in self.blocks.items(): + out_edges = [f"{i}" for i in self.out_edges[key]] + out_edges = str(out_edges).replace("\'", "\"") + back_edges = [f"{i}" for i in self.back_edges[key]] + jump_target_str= f""" + "{str(key)}": + type: {type(value)} + out: {out_edges}""" + + if back_edges: + back_edges = str(back_edges).replace("\'", "\"") + jump_target_str += f""" + back: {back_edges}""" + yaml_string += dedent(jump_target_str) + + return yaml_string + + def to_dict(self): + block_map_graph = self.graph + graph_dict = {} + for key, value in block_map_graph.items(): + curr_dict = {} + curr_dict["jt"] = [f"{i.index}" for i in value._out_edges] + if value.backedges: + curr_dict["be"] = [f"{i.index}" for i in value.backedges] + graph_dict[str(key.index)] = curr_dict + return graph_dict diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 9f13e29..cdd70fc 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -9,19 +9,19 @@ SyntheticExit, SynthenticAssignment, PythonBytecodeLabel, + BlockName ) -from numba_rvsdg.core.datastructures.block_map import BlockMap +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.basic_block import ( BasicBlock, ControlVariableBlock, BranchBlock, - RegionBlock, ) from numba_rvsdg.core.utils import _logger -def loop_restructure_helper(bbmap: BlockMap, loop: Set[Label]): +def loop_restructure_helper(bbmap: SCFG, loop: Set[Label]): """Loop Restructuring Applies the algorithm LOOP RESTRUCTURING from section 4.1 of Bahmann2015. @@ -30,8 +30,8 @@ def loop_restructure_helper(bbmap: BlockMap, loop: Set[Label]): Parameters ---------- - bbmap: BlockMap - The BlockMap containing the loop + bbmap: SCFG + The SCFG containing the loop loop: Set[Label] The loop (strongly connected components) that is to be restructured @@ -171,7 +171,7 @@ def reverse_lookup(d, value): backedges=(), variable_assignment=variable_assignment, ) - # Add the new block to the BlockMap + # Add the new block to the SCFG bbmap.add_block(synth_assign_block) # Update the jump targets again, order matters new_jt[new_jt.index(jt)] = synth_assign @@ -205,16 +205,16 @@ def reverse_lookup(d, value): bbmap.add_block(synth_exit_block) -def restructure_loop(bbmap: BlockMap): +def restructure_loop(bbmap: SCFG): """Inplace restructuring of the given graph to extract loops using strongly-connected components """ # obtain a List of Sets of Labels, where all labels in each set are strongly # connected, i.e. all reachable from one another by traversing the subset - scc: List[Set[Label]] = bbmap.compute_scc() + scc: List[Set[SCFG]] = bbmap.compute_scc() # loops are defined as strongly connected subsets who have more than a # single label and single label loops that point back to to themselves. - loops: List[Set[Label]] = [ + loops: List[Set[SCFG]] = [ nodes for nodes in scc if len(nodes) > 1 or next(iter(nodes)) in bbmap[next(iter(nodes))].jump_targets @@ -229,7 +229,7 @@ def restructure_loop(bbmap: BlockMap): extract_region(bbmap, loop, "loop") -def find_head_blocks(bbmap: BlockMap, begin: Label) -> Set[Label]: +def find_head_blocks(bbmap: SCFG, begin: Label) -> Set[Label]: head = bbmap.find_head() head_region_blocks = set() current_block = head @@ -246,7 +246,7 @@ def find_head_blocks(bbmap: BlockMap, begin: Label) -> Set[Label]: return head_region_blocks -def find_branch_regions(bbmap: BlockMap, begin: Label, end: Label) -> Set[Label]: +def find_branch_regions(bbmap: SCFG, begin: Label, end: Label) -> Set[Label]: # identify branch regions doms = _doms(bbmap) postdoms = _post_doms(bbmap) @@ -271,7 +271,7 @@ def find_branch_regions(bbmap: BlockMap, begin: Label, end: Label) -> Set[Label] return branch_regions -def _find_branch_regions(bbmap: BlockMap, begin: Label, end: Label) -> Set[Label]: +def _find_branch_regions(bbmap: SCFG, begin: Label, end: Label) -> Set[Label]: # identify branch regions branch_regions = [] for bra_start in bbmap[begin].jump_targets: @@ -281,7 +281,7 @@ def _find_branch_regions(bbmap: BlockMap, begin: Label, end: Label) -> Set[Label def find_tail_blocks( - bbmap: BlockMap, begin: Set[Label], head_region_blocks, branch_regions + bbmap: SCFG, begin: Set[Label], head_region_blocks, branch_regions ): tail_subregion = set((b for b in bbmap.graph.keys())) tail_subregion.difference_update(head_region_blocks) @@ -305,8 +305,8 @@ def extract_region(bbmap, region_blocks, region_kind): region_header = next(iter(headers)) region_exiting = next(iter(exiting_blocks)) - head_subgraph = BlockMap( - {label: bbmap.graph[label] for label in region_blocks}, clg=bbmap.clg + head_subgraph = SCFG( + {label: bbmap.graph[label] for label in region_blocks}, name_gen=bbmap.name_gen ) if isinstance(bbmap[region_exiting], RegionBlock): @@ -327,7 +327,7 @@ def extract_region(bbmap, region_blocks, region_kind): bbmap.graph[region_header] = subregion -def restructure_branch(bbmap: BlockMap): +def restructure_branch(bbmap: SCFG): print("restructure_branch", bbmap.graph) doms = _doms(bbmap) postdoms = _post_doms(bbmap) @@ -351,7 +351,7 @@ def restructure_branch(bbmap: BlockMap): # Unify headers of tail subregion if need be. headers, entries = bbmap.find_headers_and_entries(tail_region_blocks) if len(headers) > 1: - end = SyntheticHead(bbmap.clg.new_index()) + end = SyntheticHead(bbmap.name_gen.new_index()) bbmap.insert_block_and_control_blocks(end, entries, headers) # Recompute regions. @@ -376,7 +376,7 @@ def restructure_branch(bbmap: BlockMap): else: # Insert SyntheticBranch tail_headers, _ = bbmap.find_headers_and_entries(tail_region_blocks) - synthetic_branch_block_label = SyntheticBranch(str(bbmap.clg.new_index())) + synthetic_branch_block_label = SyntheticBranch(str(bbmap.name_gen.new_index())) bbmap.insert_block(synthetic_branch_block_label, (begin,), tail_headers) # Recompute regions. @@ -397,7 +397,7 @@ def restructure_branch(bbmap: BlockMap): def _iter_branch_regions( - bbmap: BlockMap, immdoms: Dict[Label, Label], postimmdoms: Dict[Label, Label] + bbmap: SCFG, immdoms: Dict[Label, Label], postimmdoms: Dict[Label, Label] ): for begin, node in [i for i in bbmap.graph.items()]: if len(node.jump_targets) > 1: @@ -428,7 +428,7 @@ def _imm_doms(doms: Dict[Label, Set[Label]]) -> Dict[Label, Label]: return out -def _doms(bbmap: BlockMap): +def _doms(bbmap: SCFG): # compute dom entries = set() preds_table = defaultdict(set) @@ -450,7 +450,7 @@ def _doms(bbmap: BlockMap): ) -def _post_doms(bbmap: BlockMap): +def _post_doms(bbmap: SCFG): # compute post dom entries = set() for k, v in bbmap.graph.items(): @@ -481,13 +481,13 @@ def _find_dominators_internal(entries, nodes, preds_table, succs_table): # in http://pages.cs.wisc.edu/~fischer/cs701.f08/finding.loops.html # if post: - # entries = set(self._exit_points) - # preds_table = self._succs - # succs_table = self._preds + # entries = set(scfg._exit_points) + # preds_table = scfg._succs + # succs_table = scfg._preds # else: - # entries = set([self._entry_point]) - # preds_table = self._preds - # succs_table = self._succs + # entries = set([scfg._entry_point]) + # preds_table = scfg._preds + # succs_table = scfg._succs import functools @@ -517,3 +517,108 @@ def _find_dominators_internal(entries, nodes, preds_table, succs_table): doms[n] = new_doms todo.extend(succs_table[n]) return doms + + +def insert_block_and_control_blocks( + scfg, label: Label, predecessors: Set[BlockName], successors: Set[BlockName] +): + # TODO: needs a diagram and documentaion + # name of the variable for this branching assignment + branch_variable = scfg.name_gen.new_name(label) + # initialize new block, which will hold the branching table + new_block = BranchBlock( + label=label, + _jump_targets=tuple(successors), + backedges=set(), + variable=branch_variable, + branch_value_table=branch_value_table, + name_gen=scfg.name_gen + ) + # initial value of the assignment + branch_variable_value = 0 + # store for the mapping from variable value to blockname + branch_value_table = {} + # Replace any arcs from any of predecessors to any of successors with + # an arc through the to be inserted block instead. + for name in predecessors: + block = scfg.graph[name] + jt = list(block.out_edges) + # Need to create synthetic assignments for each arc from a + # predecessors to a successor and insert it between the predecessor + # and the newly created block + for s in set(jt).intersection(successors): + synth_assign = SynthenticAssignment() + variable_assignment = {} + variable_assignment[branch_variable] = branch_variable_value + synth_assign_block = ControlVariableBlock( + label=synth_assign, + _jump_targets=(branch_variable,), + backedges=(), + variable_assignment=variable_assignment, + name_gen=scfg.name_gen + ) + # add block + scfg.add_block(synth_assign_block) + # update branching table + branch_value_table[branch_variable_value] = s + # update branching variable + branch_variable_value += 1 + # replace previous successor with synth_assign + jt[jt.index(s)] = synth_assign + # finally, replace the out_edges + scfg.add_block( + scfg.graph.pop(name).replace_jump_targets(out_edges=tuple(jt)) + ) + + assert new_block.block_name == branch_variable + # add block to scfg + scfg.add_block(new_block) + + +def join_returns(scfg): + """Close the CFG. + + A closed CFG is a CFG with a unique entry and exit node that have no + predescessors and no successors respectively. + """ + # for all nodes that contain a return + return_nodes = [node for node in scfg.blocks if scfg.blocks[node].is_exiting] + # close if more than one is found + if len(return_nodes) > 1: + return_solo_label = SyntheticReturn() + scfg.insert_block(return_solo_label, return_nodes, tuple()) + + +def join_tails_and_exits(scfg, tails: Set[BlockName], exits: Set[BlockName]): + if len(tails) == 1 and len(exits) == 1: + # no-op + solo_tail_name = next(iter(tails)) + solo_exit_name = next(iter(exits)) + return solo_tail_name, solo_exit_name + + if len(tails) == 1 and len(exits) == 2: + # join only exits + solo_tail_name = next(iter(tails)) + solo_exit_label = SyntheticExit() + exit_block = scfg.insert_block(solo_exit_label, tails, exits) + solo_exit_name = exit_block.block_name + return solo_tail_name, solo_exit_name + + if len(tails) >= 2 and len(exits) == 1: + # join only tails + solo_tail_label = SyntheticTail() + solo_exit_name = next(iter(exits)) + tail_block = scfg.insert_block(solo_tail_label, tails, exits) + solo_tail_name = tail_block.block_name + return solo_tail_name, solo_exit_name + + if len(tails) >= 2 and len(exits) >= 2: + # join both tails and exits + solo_tail_label = SyntheticTail() + solo_exit_label = SyntheticExit() + tail_block = scfg.insert_block(solo_tail_label, tails, exits) + solo_tail_name = tail_block.block_name + exit_block = scfg.insert_block(solo_exit_label, set((solo_tail_label,)), exits) + solo_exit_name = exit_block.block_name + return solo_tail_name, solo_exit_name + diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index bc3ac02..56395b9 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -1,12 +1,11 @@ import logging from numba_rvsdg.core.datastructures.basic_block import ( BasicBlock, - RegionBlock, PythonBytecodeBlock, ControlVariableBlock, BranchBlock, ) -from numba_rvsdg.core.datastructures.block_map import BlockMap +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.labels import ( Label, PythonBytecodeLabel, @@ -24,7 +23,7 @@ def __init__(self): self.g = Digraph() def render_region_block( - self, digraph: "Digraph", label: Label, regionblock: RegionBlock + self, digraph: "Digraph", label: Label ): # render subgraph graph = regionblock.get_full_graph() @@ -135,7 +134,7 @@ def render_byteflow(self, byteflow: ByteFlow): return self.g def bcmap_from_bytecode(self, bc: dis.Bytecode): - self.bcmap: Dict[int, dis.Instruction] = BlockMap.bcmap_from_bytecode(bc) + self.bcmap: Dict[int, dis.Instruction] = SCFG.bcmap_from_bytecode(bc) logging.basicConfig(level=logging.DEBUG) diff --git a/numba_rvsdg/tests/simulator.py b/numba_rvsdg/tests/simulator.py index 6d8e837..1cc6ab2 100644 --- a/numba_rvsdg/tests/simulator.py +++ b/numba_rvsdg/tests/simulator.py @@ -1,7 +1,7 @@ from collections import ChainMap from dis import Instruction from numba_rvsdg.core.datastructures.byte_flow import ByteFlow -from numba_rvsdg.core.datastructures.block_map import BlockMap +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.basic_block import ( BasicBlock, PythonBytecodeBlock, @@ -23,7 +23,7 @@ class Simulator: - """BlockMap simulator. + """SCFG simulator. This is a simulator utility to be used for testing. diff --git a/numba_rvsdg/tests/test_block_map.py b/numba_rvsdg/tests/test_block_map.py deleted file mode 100644 index ac07420..0000000 --- a/numba_rvsdg/tests/test_block_map.py +++ /dev/null @@ -1,22 +0,0 @@ - -from numba_rvsdg.tests.test_transforms import MapComparator -from numba_rvsdg.core.datastructures.basic_block import BasicBlock -from numba_rvsdg.core.datastructures.block_map import BlockMap -from numba_rvsdg.core.datastructures.labels import ControlLabel - -class TestBlockMapIterator(MapComparator): - - def test_block_map_iter(self): - expected = [ - (ControlLabel("0"), BasicBlock(label=ControlLabel("0"), - _jump_targets=(ControlLabel("1"),))), - (ControlLabel("1"), BasicBlock(label=ControlLabel("1"))), - ] - block_map = BlockMap.from_yaml(""" - "0": - jt: ["1"] - "1": - jt: [] - """) - received = list(block_map) - self.assertEqual(expected, received) diff --git a/numba_rvsdg/tests/test_blockmap.py b/numba_rvsdg/tests/test_blockmap.py deleted file mode 100644 index f9c8a63..0000000 --- a/numba_rvsdg/tests/test_blockmap.py +++ /dev/null @@ -1,104 +0,0 @@ - -from unittest import main -from textwrap import dedent -from numba_rvsdg.core.datastructures.block_map import BlockMap - -from numba_rvsdg.tests.test_utils import MapComparator - - -class TestBlockMapConversion(MapComparator): - - def test_yaml_conversion(self): - # Case # 1: Acyclic graph, no back-edges - cases = [""" - "0": - jt: ["1", "2"] - "1": - jt: ["3"] - "2": - jt: ["4"] - "3": - jt: ["4"] - "4": - jt: []""", - # Case # 2: Cyclic graph, no back edges - """ - "0": - jt: ["1", "2"] - "1": - jt: ["5"] - "2": - jt: ["1", "5"] - "3": - jt: ["0"] - "4": - jt: [] - "5": - jt: ["3", "4"]""", - # Case # 3: Graph with backedges - """ - "0": - jt: ["1"] - "1": - jt: ["2", "3"] - "2": - jt: ["4"] - "3": - jt: [] - "4": - jt: ["2", "3"] - be: ["2"]"""] - - for case in cases: - case = dedent(case) - block_map = BlockMap.from_yaml(case) - self.assertEqual(case, block_map.to_yaml()) - - def test_dict_conversion(self): - # Case # 1: Acyclic graph, no back-edges - cases = [{ - "0": - {"jt": ["1", "2"]}, - "1": - {"jt": ["3"]}, - "2": - {"jt": ["4"]}, - "3": - {"jt": ["4"]}, - "4": - {"jt": []}}, - # Case # 2: Cyclic graph, no back edges - { - "0": - {"jt": ["1", "2"]}, - "1": - {"jt": ["5"]}, - "2": - {"jt": ["1", "5"]}, - "3": - {"jt": ["0"]}, - "4": - {"jt": []}, - "5": - {"jt": ["3", "4"]}}, - # Case # 3: Graph with backedges - { - "0": - {"jt": ["1"]}, - "1": - {"jt": ["2", "3"]}, - "2": - {"jt": ["4"]}, - "3": - {"jt": []}, - "4": - {"jt": ["2", "3"], - "be": ["2"]}}] - - for case in cases: - block_map = BlockMap.from_dict(case) - self.assertEqual(case, block_map.to_dict()) - -if __name__ == "__main__": - main() - diff --git a/numba_rvsdg/tests/test_byteflow.py b/numba_rvsdg/tests/test_byteflow.py index 6f429a6..47bc430 100644 --- a/numba_rvsdg/tests/test_byteflow.py +++ b/numba_rvsdg/tests/test_byteflow.py @@ -2,9 +2,9 @@ import unittest from numba_rvsdg.core.datastructures.basic_block import PythonBytecodeBlock -from numba_rvsdg.core.datastructures.labels import PythonBytecodeLabel +from numba_rvsdg.core.datastructures.labels import PythonBytecodeLabel, NameGenerator, BlockName, get_label_class from numba_rvsdg.core.datastructures.byte_flow import ByteFlow -from numba_rvsdg.core.datastructures.block_map import BlockMap +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.flow_info import FlowInfo @@ -102,20 +102,22 @@ def test(self): ), ), } - received = BlockMap.bcmap_from_bytecode(bytecode) + received = ByteFlow.bcmap_from_bytecode(bytecode) self.assertEqual(expected, received) class TestPythonBytecodeBlock(unittest.TestCase): def test_constructor(self): + name_gen = NameGenerator(index=0) block = PythonBytecodeBlock( - label=PythonBytecodeLabel(index=0), + label=PythonBytecodeLabel(), begin=0, end=8, _jump_targets=(), backedges=(), + name_gen=name_gen ) - self.assertEqual(block.label, PythonBytecodeLabel(index=0)) + self.assertEqual(block.label, PythonBytecodeLabel()) self.assertEqual(block.begin, 0) self.assertEqual(block.end, 8) self.assertFalse(block.fallthrough) @@ -123,26 +125,36 @@ def test_constructor(self): self.assertEqual(block.jump_targets, ()) self.assertEqual(block.backedges, ()) + block_name = BlockName('pythonbytecodelabel_0') + self.assertEqual(block.block_name, block_name) + def test_is_jump_target(self): + name_gen = NameGenerator(index=0) + label = PythonBytecodeLabel() + block = PythonBytecodeBlock( - label=PythonBytecodeLabel(index=0), + label=label, begin=0, end=8, - _jump_targets=(PythonBytecodeLabel(index=1),), + _jump_targets=(name_gen.new_block_name(label),), backedges=(), + name_gen=name_gen ) - self.assertEqual(block.jump_targets, (PythonBytecodeLabel(index=1),)) + self.assertEqual(block.block_name, BlockName(name='pythonbytecodelabel_1')) + self.assertEqual(block.jump_targets, (BlockName(name='pythonbytecodelabel_0'),)) self.assertFalse(block.is_exiting) def test_get_instructions(self): # If the function definition line changes, just change the variable below, rest of it will adjust as long as function remains the same func_def_line = 11 + name_gen = NameGenerator(index=0) block = PythonBytecodeBlock( - label=PythonBytecodeLabel(index=0), + label=PythonBytecodeLabel(), begin=0, end=8, _jump_targets=(), backedges=(), + name_gen=name_gen ) expected = [ Instruction( @@ -211,7 +223,7 @@ def test_get_instructions(self): ), ] - received = block.get_instructions(BlockMap.bcmap_from_bytecode(bytecode)) + received = block.get_instructions(SCFG.bcmap_from_bytecode(bytecode)) self.assertEqual(expected, received) @@ -229,17 +241,20 @@ def test_from_bytecode(self): self.assertEqual(expected, received) def test_build_basic_blocks(self): - expected = BlockMap( - graph={ - PythonBytecodeLabel(index=0): PythonBytecodeBlock( - label=PythonBytecodeLabel(index=0), + name_gen = NameGenerator(index=0) + expected = SCFG( + graph={} + ) + block = PythonBytecodeBlock( + label=PythonBytecodeLabel(), begin=0, end=10, _jump_targets=(), backedges=(), + name_gen=name_gen ) - } - ) + expected.add_block(block) + received = FlowInfo.from_bytecode(bytecode).build_basicblocks() self.assertEqual(expected, received) @@ -251,17 +266,9 @@ def test_constructor(self): self.assertEqual(len(byteflow.bbmap), 0) def test_from_bytecode(self): - bbmap = BlockMap( - graph={ - PythonBytecodeLabel(index=0): PythonBytecodeBlock( - label=PythonBytecodeLabel(index=0), - begin=0, - end=10, - _jump_targets=(), - backedges=(), - ) - } - ) + bbmap = SCFG() + + bbmap.add_block('python_bytecode', begin=0, end=10, label=get_label_class('python_bytecode')()) expected = ByteFlow(bc=bytecode, bbmap=bbmap) received = ByteFlow.from_bytecode(fun) self.assertEqual(expected.bbmap, received.bbmap) diff --git a/numba_rvsdg/tests/test_scfg.py b/numba_rvsdg/tests/test_scfg.py new file mode 100644 index 0000000..37cb46c --- /dev/null +++ b/numba_rvsdg/tests/test_scfg.py @@ -0,0 +1,161 @@ + +from unittest import main +from textwrap import dedent +from numba_rvsdg.core.datastructures.scfg import SCFG + +from numba_rvsdg.tests.test_utils import MapComparator +from numba_rvsdg.core.datastructures.basic_block import BasicBlock +from numba_rvsdg.core.datastructures.labels import Label, NameGenerator + + +class TestBlockMapConversion(MapComparator): + + def test_yaml_conversion(self): + # Case # 1: Acyclic graph, no back-edges + cases = [""" + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["3"] + "2": + type: "basic" + out: ["4"] + "3": + type: "basic" + out: ["4"] + "4": + type: "basic" + out: []""", + # Case # 2: Cyclic graph, no back edges + """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["5"] + "2": + type: "basic" + out: ["1", "5"] + "3": + type: "basic" + out: ["0"] + "4": + type: "basic" + out: [] + "5": + type: "basic" + out: ["3", "4"]""", + # Case # 3: Graph with backedges + """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: ["2", "3"] + "2": + type: "basic" + out: ["4"] + "3": + type: "basic" + out: [] + "4": + type: "basic" + out: ["2", "3"] + back: ["2"]"""] + + for case in cases: + case = dedent(case) + block_map, ref_dict = SCFG.from_yaml(case) + # TODO: use ref_dict for comparision + self.assertEqual(case, block_map.to_yaml()) + + def test_dict_conversion(self): + # Case # 1: Acyclic graph, no back-edges + cases = [{ + "0": + {"type": "basic", + "out": ["1", "2"]}, + "1": + {"type": "basic", + "out": ["3"]}, + "2": + {"type": "basic", + "out": ["4"]}, + "3": + {"type": "basic", + "out": ["4"]}, + "4": + {"type": "basic", + "out": []}}, + # Case # 2: Cyclic graph, no back edges + { + "0": + {"type": "basic", + "out": ["1", "2"]}, + "1": + {"type": "basic", + "out": ["5"]}, + "2": + {"type": "basic", + "out": ["1", "5"]}, + "3": + {"type": "basic", + "out": ["0"]}, + "4": + {"type": "basic", + "out": []}, + "5": + {"type": "basic", + "out": ["3", "4"]}}, + # Case # 3: Graph with backedges + { + "0": + {"type": "basic", + "out": ["1"]}, + "1": + {"type": "basic", + "out": ["2", "3"]}, + "2": + {"type": "basic", + "out": ["4"]}, + "3": + {"type": "basic", + "out": []}, + "4": + {"type": "basic", + "out": ["2", "3"], + "back": ["2"]} + }] + + for case in cases: + block_map = SCFG.from_dict(case) + self.assertEqual(case, block_map.to_dict()) + + +class TestSCFGIterator(MapComparator): + + def test_scfg_iter(self): + name_generator = NameGenerator() + block_0 = BasicBlock(name_generator, Label()) + block_1 = BasicBlock(name_generator, Label()) + expected = [ + (block_0.block_name, block_0), + (block_1.block_name, block_1), + ] + scfg = SCFG.from_yaml(""" + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + """) + received = list(scfg) + self.assertEqual(expected, received) + +if __name__ == "__main__": + main() + diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index 7c1d621..7ff7d48 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -6,7 +6,7 @@ SyntheticTail, SyntheticExit, ) -from numba_rvsdg.core.datastructures.block_map import BlockMap, wrap_id +from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.transformations import loop_restructure_helper from numba_rvsdg.tests.test_utils import MapComparator @@ -19,7 +19,7 @@ def test_linear(self): "1": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["2"] @@ -28,11 +28,11 @@ def test_linear(self): "2": jt: ["1"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) original_block_map.insert_block( ControlLabel("2"), wrap_id(("0",)), wrap_id(("1",)) ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) def test_dual_predecessor(self): original = """ @@ -43,7 +43,7 @@ def test_dual_predecessor(self): "2": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["3"] @@ -54,11 +54,11 @@ def test_dual_predecessor(self): "3": jt: ["2"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) original_block_map.insert_block( ControlLabel("3"), wrap_id(("0", "1")), wrap_id(("2",)) ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) def test_dual_successor(self): original = """ @@ -69,7 +69,7 @@ def test_dual_successor(self): "2": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["3"] @@ -80,13 +80,13 @@ def test_dual_successor(self): "3": jt: ["1", "2"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) original_block_map.insert_block( ControlLabel("3"), wrap_id(("0",)), wrap_id(("1", "2")), ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) def test_dual_predecessor_and_dual_successor(self): original = """ @@ -101,7 +101,7 @@ def test_dual_predecessor_and_dual_successor(self): "4": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -116,13 +116,13 @@ def test_dual_predecessor_and_dual_successor(self): "5": jt: ["3", "4"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) original_block_map.insert_block( ControlLabel("5"), wrap_id(("1", "2")), wrap_id(("3", "4")), ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): original = """ @@ -137,7 +137,7 @@ def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): "4": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -152,13 +152,13 @@ def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): "5": jt: ["3", "4"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) original_block_map.insert_block( ControlLabel("5"), wrap_id(("1", "2")), wrap_id(("3", "4")), ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) class TestJoinReturns(MapComparator): @@ -171,7 +171,7 @@ def test_two_returns(self): "2": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -182,7 +182,7 @@ def test_two_returns(self): "3": jt: [] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) original_block_map.join_returns() self.assertMapEqual(expected_block_map, original_block_map) @@ -195,14 +195,14 @@ def test_join_tails_and_exits_case_00(self): "1": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1"] "1": jt: [] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) tails = wrap_id(("0",)) exits = wrap_id(("1",)) @@ -210,7 +210,7 @@ def test_join_tails_and_exits_case_00(self): tails, exits ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) self.assertEqual(ControlLabel("0"), solo_tail_label) self.assertEqual(ControlLabel("1"), solo_exit_label) @@ -225,7 +225,7 @@ def test_join_tails_and_exits_case_01(self): "3": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["4"] @@ -238,7 +238,7 @@ def test_join_tails_and_exits_case_01(self): "4": jt: ["1", "2"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) tails = wrap_id(("0",)) exits = wrap_id(("1", "2")) @@ -246,7 +246,7 @@ def test_join_tails_and_exits_case_01(self): tails, exits ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) self.assertEqual(ControlLabel("0"), solo_tail_label) self.assertEqual(SyntheticExit("4"), solo_exit_label) @@ -261,7 +261,7 @@ def test_join_tails_and_exits_case_02_01(self): "3": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -274,7 +274,7 @@ def test_join_tails_and_exits_case_02_01(self): "4": jt: ["3"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) tails = wrap_id(("1", "2")) exits = wrap_id(("3",)) @@ -282,7 +282,7 @@ def test_join_tails_and_exits_case_02_01(self): tails, exits ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) self.assertEqual(SyntheticTail("4"), solo_tail_label) self.assertEqual(ControlLabel("3"), solo_exit_label) @@ -297,7 +297,7 @@ def test_join_tails_and_exits_case_02_02(self): "3": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -310,15 +310,15 @@ def test_join_tails_and_exits_case_02_02(self): "4": jt: ["3"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) tails = wrap_id(("1", "2")) exits = wrap_id(("3",)) - solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( + solo_tail_label, solo_exit_label = original_scfg.join_tails_and_exits( tails, exits ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) self.assertEqual(SyntheticTail("4"), solo_tail_label) self.assertEqual(ControlLabel("3"), solo_exit_label) @@ -338,7 +338,7 @@ def test_join_tails_and_exits_case_03_01(self): "5": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -357,14 +357,14 @@ def test_join_tails_and_exits_case_03_01(self): "7": jt: ["3", "4"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) tails = wrap_id(("1", "2")) exits = wrap_id(("3", "4")) solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( tails, exits ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) self.assertEqual(SyntheticTail("6"), solo_tail_label) self.assertEqual(SyntheticExit("7"), solo_exit_label) @@ -384,7 +384,7 @@ def test_join_tails_and_exits_case_03_02(self): "5": jt: [] """ - original_block_map = BlockMap.from_yaml(original) + original_block_map = SCFG.from_yaml(original) expected = """ "0": jt: ["1", "2"] @@ -403,13 +403,13 @@ def test_join_tails_and_exits_case_03_02(self): "7": jt: ["3", "4"] """ - expected_block_map = BlockMap.from_yaml(expected) + expected_block_map = SCFG.from_yaml(expected) tails = wrap_id(("1", "2")) exits = wrap_id(("3", "4")) solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( tails, exits ) - self.assertMapEqual(expected_block_map, original_block_map) + self.assertMapEqual(expected_scfg, original_scfg) self.assertEqual(SyntheticTail("6"), solo_tail_label) self.assertEqual(SyntheticExit("7"), solo_exit_label) @@ -435,8 +435,8 @@ def test_no_op_mono(self): "2": jt: [] """ - original_block_map = BlockMap.from_yaml(original) - expected_block_map = BlockMap.from_yaml(expected) + original_block_map = SCFG.from_yaml(original) + expected_block_map = SCFG.from_yaml(expected) loop_restructure_helper(original_block_map, set(wrap_id({"1"}))) self.assertMapEqual(expected_block_map, original_block_map) @@ -463,8 +463,8 @@ def test_no_op(self): "3": jt: [] """ - original_block_map = BlockMap.from_yaml(original) - expected_block_map = BlockMap.from_yaml(expected) + original_block_map = SCFG.from_yaml(original) + expected_block_map = SCFG.from_yaml(expected) loop_restructure_helper(original_block_map, set(wrap_id({"1", "2"}))) self.assertMapEqual(expected_block_map, original_block_map) @@ -500,8 +500,8 @@ def test_backedge_not_exiting(self): "6": jt: ["4"] """ - original_block_map = BlockMap.from_yaml(original) - expected_block_map = BlockMap.from_yaml(expected) + original_block_map = SCFG.from_yaml(original) + expected_block_map = SCFG.from_yaml(expected) loop_restructure_helper(original_block_map, set(wrap_id({"1", "2"}))) self.assertMapEqual(expected_block_map, original_block_map) @@ -544,8 +544,8 @@ def test_double_exit(self): "8": jt: ["5"] """ - original_block_map = BlockMap.from_yaml(original) - expected_block_map = BlockMap.from_yaml(expected) + original_block_map = SCFG.from_yaml(original) + expected_block_map = SCFG.from_yaml(expected) loop_restructure_helper(original_block_map, set(wrap_id({"1", "2", "3"}))) self.assertMapEqual(expected_block_map, original_block_map) @@ -595,8 +595,8 @@ def test_double_header(self): "12": jt: ["9"] """ - original_block_map = BlockMap.from_yaml(original) - expected_block_map = BlockMap.from_yaml(expected) + original_block_map = SCFG.from_yaml(original) + expected_block_map = SCFG.from_yaml(expected) loop_restructure_helper(original_block_map, set(wrap_id({"1", "2", "3", "4"}))) self.assertMapEqual(expected_block_map, original_block_map) @@ -662,8 +662,8 @@ def test_double_header_double_exiting(self): "16": jt: ["11"] """ - original_block_map = BlockMap.from_yaml(original) - expected_block_map = BlockMap.from_yaml(expected) + original_block_map = SCFG.from_yaml(original) + expected_block_map = SCFG.from_yaml(expected) loop_restructure_helper(original_block_map, set(wrap_id({"1", "2", "3", "4"}))) self.assertMapEqual(expected_block_map, original_block_map) From e5e0821476edf8979606aa9cfb7407c608c4e0c3 Mon Sep 17 00:00:00 2001 From: kc611 Date: Mon, 27 Mar 2023 00:50:49 +0530 Subject: [PATCH 02/30] Refactored renderer and simulator --- numba_rvsdg/core/datastructures/byte_flow.py | 18 +- numba_rvsdg/core/datastructures/flow_info.py | 6 +- numba_rvsdg/core/datastructures/scfg.py | 33 ++-- numba_rvsdg/core/transformations.py | 197 ++++++++++--------- numba_rvsdg/rendering/rendering.py | 179 ++++++++--------- numba_rvsdg/tests/simulator.py | 138 ++++--------- numba_rvsdg/tests/test_byteflow.py | 10 +- numba_rvsdg/tests/test_fig3.py | 4 +- numba_rvsdg/tests/test_fig4.py | 4 +- numba_rvsdg/tests/test_simulate.py | 39 +--- 10 files changed, 271 insertions(+), 357 deletions(-) diff --git a/numba_rvsdg/core/datastructures/byte_flow.py b/numba_rvsdg/core/datastructures/byte_flow.py index 3fe38a7..412729f 100644 --- a/numba_rvsdg/core/datastructures/byte_flow.py +++ b/numba_rvsdg/core/datastructures/byte_flow.py @@ -11,7 +11,7 @@ @dataclass(frozen=True) class ByteFlow: bc: dis.Bytecode - bbmap: SCFG + scfg: SCFG @staticmethod def from_bytecode(code) -> "ByteFlow": @@ -19,25 +19,25 @@ def from_bytecode(code) -> "ByteFlow": _logger.debug("Bytecode\n%s", _LogWrap(lambda: bc.dis())) flowinfo = FlowInfo.from_bytecode(bc) - bbmap = flowinfo.build_basicblocks() - return ByteFlow(bc=bc, bbmap=bbmap) + scfg = flowinfo.build_basicblocks() + return ByteFlow(bc=bc, scfg=scfg) def _join_returns(self): - join_returns(self.bbmap) + join_returns(self.scfg) def _restructure_loop(self): - restructure_loop(self.bbmap) + restructure_loop(self.scfg) def _restructure_branch(self): - restructure_branch(self.bbmap) + restructure_branch(self.scfg) def restructure(self): # close - join_returns(self.bbmap) + join_returns(self.scfg) # handle loop - restructure_loop(self.bbmap) + restructure_loop(self.scfg) # handle branch - restructure_branch(self.bbmap) + restructure_branch(self.scfg) @staticmethod def bcmap_from_bytecode(bc: dis.Bytecode): diff --git a/numba_rvsdg/core/datastructures/flow_info.py b/numba_rvsdg/core/datastructures/flow_info.py index 3ead8d8..7ba8730 100644 --- a/numba_rvsdg/core/datastructures/flow_info.py +++ b/numba_rvsdg/core/datastructures/flow_info.py @@ -77,10 +77,10 @@ def build_basicblocks(self: "FlowInfo", end_offset=None) -> "SCFG": end_offset = _next_inst_offset(self.last_offset) for begin, end in zip(offsets, [*offsets[1:], end_offset]): - names[begin] = scfg.add_block(block_type=PythonBytecodeBlock, + names[begin] = scfg.add_block(block_type='python_bytecode', + block_label=PythonBytecodeLabel(), begin=begin, - end=end, - label=PythonBytecodeLabel()) + end=end) for begin, end in zip(offsets, [*offsets[1:], end_offset]): targets: Tuple[BlockName, ...] diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 21691d5..3d09834 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -12,6 +12,7 @@ ) from numba_rvsdg.core.datastructures.region import Region from numba_rvsdg.core.datastructures.labels import ( + Label, BlockName, NameGenerator, RegionName, @@ -197,6 +198,12 @@ def is_reachable_dfs(self, begin: BlockName, end: BlockName): # -> TypeGuard: if block in self.blocks: to_vist.extend(self.out_edges[self.blocks[block]]) + def is_exiting(self, block_name: BlockName): + return len(self.out_edges[block_name]) == 0 + + def is_fallthrough(self, block_name: BlockName): + return len(self.out_edges[block_name]) == 1 + def check_graph(self): pass @@ -209,10 +216,11 @@ def check_graph(self): # del self.in_edges[name] # self.check_graph() - def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], block_type: type = BasicBlock, **block_args): + def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], block_type: str = 'basic', block_label: Label = Label(), **block_args): # TODO: needs a diagram and documentaion # initialize new block + block_type = get_block_class(block_type) new_block_name = self.add_block(block_type, **block_args) # Replace any arcs from any of predecessors to any of successors with # an arc through the inserted block instead. @@ -238,10 +246,11 @@ def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], self.check_graph() return new_block_name - def add_block(self, block_type: str = 'basic', **block_args): + def add_block(self, block_type: str = 'basic', block_label: Label = Label(), **block_args): block_type = get_block_class(block_type) new_block = block_type( **block_args, + label=block_label, name_gen=self.name_gen ) @@ -316,13 +325,13 @@ def to_yaml(self): return yaml_string - def to_dict(self): - block_map_graph = self.graph - graph_dict = {} - for key, value in block_map_graph.items(): - curr_dict = {} - curr_dict["jt"] = [f"{i.index}" for i in value._out_edges] - if value.backedges: - curr_dict["be"] = [f"{i.index}" for i in value.backedges] - graph_dict[str(key.index)] = curr_dict - return graph_dict + # def to_dict(self): + # block_map_graph = self.graph + # graph_dict = {} + # for key, value in block_map_graph.items(): + # curr_dict = {} + # curr_dict["jt"] = [f"{i.index}" for i in value._out_edges] + # if value.backedges: + # curr_dict["be"] = [f"{i.index}" for i in value.backedges] + # graph_dict[str(key.index)] = curr_dict + # return graph_dict diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index cdd70fc..83e6f40 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -7,6 +7,7 @@ SyntheticHead, SyntheticExitingLatch, SyntheticExit, + SyntheticReturn, SynthenticAssignment, PythonBytecodeLabel, BlockName @@ -21,24 +22,24 @@ from numba_rvsdg.core.utils import _logger -def loop_restructure_helper(bbmap: SCFG, loop: Set[Label]): +def loop_restructure_helper(scfg: SCFG, loop: Set[Label]): """Loop Restructuring Applies the algorithm LOOP RESTRUCTURING from section 4.1 of Bahmann2015. - Note that this will modify both the `bbmap` and the `loop` in-place. + Note that this will modify both the `scfg` and the `loop` in-place. Parameters ---------- - bbmap: SCFG + scfg: SCFG The SCFG containing the loop loop: Set[Label] The loop (strongly connected components) that is to be restructured """ - headers, entries = bbmap.find_headers_and_entries(loop) - exiting_blocks, exit_blocks = bbmap.find_exiting_and_exits(loop) + headers, entries = scfg.find_headers_and_entries(loop) + exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(loop) #assert len(entries) == 1 headers_were_unified = False @@ -46,8 +47,8 @@ def loop_restructure_helper(bbmap: SCFG, loop: Set[Label]): # such that only a single loop header remains. if len(headers) > 1: headers_were_unified = True - solo_head_label = SyntheticHead(str(bbmap.clg.new_index())) - bbmap.insert_block_and_control_blocks(solo_head_label, entries, headers) + solo_head_label = SyntheticHead(str(scfg.clg.new_index())) + scfg.insert_block_and_control_blocks(solo_head_label, entries, headers) loop.add(solo_head_label) loop_head: Label = solo_head_label else: @@ -56,32 +57,32 @@ def loop_restructure_helper(bbmap: SCFG, loop: Set[Label]): # backedge to the loop header) we can exit early, since the condition for # SCFG is fullfilled. backedge_blocks = [ - block for block in loop if headers.intersection(bbmap[block].jump_targets) + block for block in loop if headers.intersection(scfg[block].jump_targets) ] if (len(backedge_blocks) == 1 and len(exiting_blocks) == 1 and backedge_blocks[0] == next(iter(exiting_blocks))): - bbmap.add_block(bbmap.graph.pop(backedge_blocks[0]).replace_backedge(loop_head)) + scfg.add_block(scfg.graph.pop(backedge_blocks[0]).replace_backedge(loop_head)) return # The synthetic exiting latch and synthetic exit need to be created # based on the state of the cfg. If there are multiple exits, we need a # SyntheticExit, otherwise we only need a SyntheticExitingLatch - synth_exiting_latch = SyntheticExitingLatch(str(bbmap.clg.new_index())) + synth_exiting_latch = SyntheticExitingLatch(str(scfg.clg.new_index())) # Set a flag, this will determine the variable assignment and block # insertion later on needs_synth_exit = len(exit_blocks) > 1 if needs_synth_exit: - synth_exit = SyntheticExit(str(bbmap.clg.new_index())) + synth_exit = SyntheticExit(str(scfg.clg.new_index())) # This sets up the various control variables. # If there were multiple headers, we must re-use the variable that was used # for looping as the exit variable if headers_were_unified: - exit_variable = bbmap[solo_head_label].variable + exit_variable = scfg[solo_head_label].variable else: - exit_variable = bbmap.clg.new_variable() + exit_variable = scfg.clg.new_variable() # This variable denotes the backedge - backedge_variable = bbmap.clg.new_variable() + backedge_variable = scfg.clg.new_variable() # Now we setup the lookup tables for the various control variables, # depending on the state of the CFG and what is needed exit_value_table = dict(((i, j) for i, j in enumerate(exit_blocks))) @@ -90,7 +91,7 @@ def loop_restructure_helper(bbmap: SCFG, loop: Set[Label]): else: backedge_value_table = dict((i, j) for i, j in enumerate((loop_head, next(iter(exit_blocks))))) if headers_were_unified: - header_value_table = bbmap[solo_head_label].branch_value_table + header_value_table = scfg[solo_head_label].branch_value_table else: header_value_table = {} @@ -107,19 +108,19 @@ def reverse_lookup(d, value): # on what is needed # All new blocks are recorded for later insertion into the loop set new_blocks = set() - doms = _doms(bbmap) + doms = _doms(scfg) # For every block in the loop: for label in sorted(loop, key=lambda x: x.index): # If the block is an exiting block or a backedge block if label in exiting_blocks or label in backedge_blocks: # Copy the jump targets, these will be modified - new_jt = list(bbmap[label].jump_targets) + new_jt = list(scfg[label].jump_targets) # For each jump_target in the blockj - for jt in bbmap[label].jump_targets: + for jt in scfg[label].jump_targets: # If the target is an exit block if jt in exit_blocks: # Create a new assignment label and record it - synth_assign = SynthenticAssignment(str(bbmap.clg.new_index())) + synth_assign = SynthenticAssignment(str(scfg.clg.new_index())) new_blocks.add(synth_assign) # Setup the table for the variable assignment variable_assignment = {} @@ -137,7 +138,7 @@ def reverse_lookup(d, value): variable_assignment=variable_assignment, ) # Insert the assignment to the block map - bbmap.add_block(synth_assign_block) + scfg.add_block(synth_assign_block) # Insert the new block into the new jump_targets making # sure, that it replaces the correct jump_target, order # matters in this case. @@ -145,7 +146,7 @@ def reverse_lookup(d, value): # If the target is the loop_head elif jt in headers and label not in doms[jt]: # Create the assignment and record it - synth_assign = SynthenticAssignment(str(bbmap.clg.new_index())) + synth_assign = SynthenticAssignment(str(scfg.clg.new_index())) new_blocks.add(synth_assign) # Setup the variables in the assignment table to point to # the correct blocks @@ -157,12 +158,12 @@ def reverse_lookup(d, value): # that point to the headers, no need to add a backedge, # since it will be contained in the SyntheticExitingLatch # later on. - block = bbmap.graph.pop(label) + block = scfg.graph.pop(label) jts = list(block.jump_targets) for h in headers: if h in jts: jts.remove(h) - bbmap.add_block(block.replace_jump_targets(jump_targets=tuple(jts))) + scfg.add_block(block.replace_jump_targets(jump_targets=tuple(jts))) # Setup the assignment block and initialize it with the # correct jump_targets and variable assignment. synth_assign_block = ControlVariableBlock( @@ -172,12 +173,12 @@ def reverse_lookup(d, value): variable_assignment=variable_assignment, ) # Add the new block to the SCFG - bbmap.add_block(synth_assign_block) + scfg.add_block(synth_assign_block) # Update the jump targets again, order matters new_jt[new_jt.index(jt)] = synth_assign # finally, replace the jump_targets for this block with the new ones - bbmap.add_block( - bbmap.graph.pop(label).replace_jump_targets(jump_targets=tuple(new_jt)) + scfg.add_block( + scfg.graph.pop(label).replace_jump_targets(jump_targets=tuple(new_jt)) ) # Add any new blocks to the loop. loop.update(new_blocks) @@ -191,8 +192,8 @@ def reverse_lookup(d, value): branch_value_table=backedge_value_table, ) loop.add(synth_exiting_latch) - bbmap.add_block(synth_exiting_latch_block) - # If an exit is to be created, we do so too, but only add it to the bbmap, + scfg.add_block(synth_exiting_latch_block) + # If an exit is to be created, we do so too, but only add it to the scfg, # since it isn't part of the loop if needs_synth_exit: synth_exit_block = BranchBlock( @@ -202,35 +203,35 @@ def reverse_lookup(d, value): variable=exit_variable, branch_value_table=exit_value_table, ) - bbmap.add_block(synth_exit_block) + scfg.add_block(synth_exit_block) -def restructure_loop(bbmap: SCFG): +def restructure_loop(scfg: SCFG): """Inplace restructuring of the given graph to extract loops using strongly-connected components """ # obtain a List of Sets of Labels, where all labels in each set are strongly # connected, i.e. all reachable from one another by traversing the subset - scc: List[Set[SCFG]] = bbmap.compute_scc() + scc: List[Set[SCFG]] = scfg.compute_scc() # loops are defined as strongly connected subsets who have more than a # single label and single label loops that point back to to themselves. loops: List[Set[SCFG]] = [ nodes for nodes in scc - if len(nodes) > 1 or next(iter(nodes)) in bbmap[next(iter(nodes))].jump_targets + if len(nodes) > 1 or next(iter(nodes)) in scfg[next(iter(nodes))].jump_targets ] _logger.debug( - "restructure_loop found %d loops in %s", len(loops), bbmap.graph.keys() + "restructure_loop found %d loops in %s", len(loops), scfg.graph.keys() ) # rotate and extract loop for loop in loops: - loop_restructure_helper(bbmap, loop) - extract_region(bbmap, loop, "loop") + loop_restructure_helper(scfg, loop) + extract_region(scfg, loop, "loop") -def find_head_blocks(bbmap: SCFG, begin: Label) -> Set[Label]: - head = bbmap.find_head() +def find_head_blocks(scfg: SCFG, begin: Label) -> Set[Label]: + head = scfg.find_head() head_region_blocks = set() current_block = head # Start at the head block and traverse the graph linearly until @@ -240,23 +241,23 @@ def find_head_blocks(bbmap: SCFG, begin: Label) -> Set[Label]: if current_block == begin: break else: - jt = bbmap.graph[current_block].jump_targets + jt = scfg.graph[current_block].jump_targets assert len(jt) == 1 current_block = next(iter(jt)) return head_region_blocks -def find_branch_regions(bbmap: SCFG, begin: Label, end: Label) -> Set[Label]: +def find_branch_regions(scfg: SCFG, begin: Label, end: Label) -> Set[Label]: # identify branch regions - doms = _doms(bbmap) - postdoms = _post_doms(bbmap) + doms = _doms(scfg) + postdoms = _post_doms(scfg) postimmdoms = _imm_doms(postdoms) immdoms = _imm_doms(doms) branch_regions = [] - jump_targets = bbmap.graph[begin].jump_targets + jump_targets = scfg.graph[begin].jump_targets for bra_start in jump_targets: for jt in jump_targets: - if jt != bra_start and bbmap.is_reachable_dfs(jt, bra_start): + if jt != bra_start and scfg.is_reachable_dfs(jt, bra_start): branch_regions.append(tuple()) break else: @@ -271,19 +272,19 @@ def find_branch_regions(bbmap: SCFG, begin: Label, end: Label) -> Set[Label]: return branch_regions -def _find_branch_regions(bbmap: SCFG, begin: Label, end: Label) -> Set[Label]: +def _find_branch_regions(scfg: SCFG, begin: Label, end: Label) -> Set[Label]: # identify branch regions branch_regions = [] - for bra_start in bbmap[begin].jump_targets: + for bra_start in scfg[begin].jump_targets: region = [] region.append(bra_start) return branch_regions def find_tail_blocks( - bbmap: SCFG, begin: Set[Label], head_region_blocks, branch_regions + scfg: SCFG, begin: Set[Label], head_region_blocks, branch_regions ): - tail_subregion = set((b for b in bbmap.graph.keys())) + tail_subregion = set((b for b in scfg.graph.keys())) tail_subregion.difference_update(head_region_blocks) for reg in branch_regions: if not reg: @@ -297,43 +298,43 @@ def find_tail_blocks( return tail_subregion -def extract_region(bbmap, region_blocks, region_kind): - headers, entries = bbmap.find_headers_and_entries(region_blocks) - exiting_blocks, exit_blocks = bbmap.find_exiting_and_exits(region_blocks) +def extract_region(scfg, region_blocks, region_kind): + headers, entries = scfg.find_headers_and_entries(region_blocks) + exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(region_blocks) assert len(headers) == 1 assert len(exiting_blocks) == 1 region_header = next(iter(headers)) region_exiting = next(iter(exiting_blocks)) head_subgraph = SCFG( - {label: bbmap.graph[label] for label in region_blocks}, name_gen=bbmap.name_gen + {label: scfg.graph[label] for label in region_blocks}, name_gen=scfg.name_gen ) - if isinstance(bbmap[region_exiting], RegionBlock): - region_exit = bbmap[region_exiting].exit + if isinstance(scfg[region_exiting], RegionBlock): + region_exit = scfg[region_exiting].exit else: region_exit = region_exiting subregion = RegionBlock( label=region_header, - _jump_targets=bbmap[region_exiting].jump_targets, + _jump_targets=scfg[region_exiting].jump_targets, backedges=(), kind=region_kind, headers=headers, subregion=head_subgraph, exit=region_exit, ) - bbmap.remove_blocks(region_blocks) - bbmap.graph[region_header] = subregion + scfg.remove_blocks(region_blocks) + scfg.graph[region_header] = subregion -def restructure_branch(bbmap: SCFG): - print("restructure_branch", bbmap.graph) - doms = _doms(bbmap) - postdoms = _post_doms(bbmap) +def restructure_branch(scfg: SCFG): + print("restructure_branch", scfg.graph) + doms = _doms(scfg) + postdoms = _post_doms(scfg) postimmdoms = _imm_doms(postdoms) immdoms = _imm_doms(doms) - regions = [r for r in _iter_branch_regions(bbmap, immdoms, postimmdoms)] + regions = [r for r in _iter_branch_regions(scfg, immdoms, postimmdoms)] # Early exit when no branching regions are found. # TODO: the whole graph should become a linear mono head @@ -342,23 +343,23 @@ def restructure_branch(bbmap: SCFG): # Compute initial regions. begin, end = regions[0] - head_region_blocks = find_head_blocks(bbmap, begin) - branch_regions = find_branch_regions(bbmap, begin, end) + head_region_blocks = find_head_blocks(scfg, begin) + branch_regions = find_branch_regions(scfg, begin, end) tail_region_blocks = find_tail_blocks( - bbmap, begin, head_region_blocks, branch_regions + scfg, begin, head_region_blocks, branch_regions ) # Unify headers of tail subregion if need be. - headers, entries = bbmap.find_headers_and_entries(tail_region_blocks) + headers, entries = scfg.find_headers_and_entries(tail_region_blocks) if len(headers) > 1: - end = SyntheticHead(bbmap.name_gen.new_index()) - bbmap.insert_block_and_control_blocks(end, entries, headers) + end = SyntheticHead(scfg.name_gen.new_index()) + scfg.insert_block_and_control_blocks(end, entries, headers) # Recompute regions. - head_region_blocks = find_head_blocks(bbmap, begin) - branch_regions = find_branch_regions(bbmap, begin, end) + head_region_blocks = find_head_blocks(scfg, begin) + branch_regions = find_branch_regions(scfg, begin, end) tail_region_blocks = find_tail_blocks( - bbmap, begin, head_region_blocks, branch_regions + scfg, begin, head_region_blocks, branch_regions ) # Branch region processing: @@ -369,37 +370,37 @@ def restructure_branch(bbmap: SCFG): bra_start, inner_nodes = region if inner_nodes: # Insert SyntheticTail - exiting_blocks, _ = bbmap.find_exiting_and_exits(inner_nodes) - tail_headers, _ = bbmap.find_headers_and_entries(tail_region_blocks) - _, _ = bbmap.join_tails_and_exits(exiting_blocks, tail_headers) + exiting_blocks, _ = scfg.find_exiting_and_exits(inner_nodes) + tail_headers, _ = scfg.find_headers_and_entries(tail_region_blocks) + _, _ = scfg.join_tails_and_exits(exiting_blocks, tail_headers) else: # Insert SyntheticBranch - tail_headers, _ = bbmap.find_headers_and_entries(tail_region_blocks) - synthetic_branch_block_label = SyntheticBranch(str(bbmap.name_gen.new_index())) - bbmap.insert_block(synthetic_branch_block_label, (begin,), tail_headers) + tail_headers, _ = scfg.find_headers_and_entries(tail_region_blocks) + synthetic_branch_block_label = SyntheticBranch(str(scfg.name_gen.new_index())) + scfg.insert_block(synthetic_branch_block_label, (begin,), tail_headers) # Recompute regions. - head_region_blocks = find_head_blocks(bbmap, begin) - branch_regions = find_branch_regions(bbmap, begin, end) + head_region_blocks = find_head_blocks(scfg, begin) + branch_regions = find_branch_regions(scfg, begin, end) tail_region_blocks = find_tail_blocks( - bbmap, begin, head_region_blocks, branch_regions + scfg, begin, head_region_blocks, branch_regions ) # extract subregions - extract_region(bbmap, head_region_blocks, "head") + extract_region(scfg, head_region_blocks, "head") for region in branch_regions: if region: bra_start, inner_nodes = region if inner_nodes: - extract_region(bbmap, inner_nodes, "branch") - extract_region(bbmap, tail_region_blocks, "tail") + extract_region(scfg, inner_nodes, "branch") + extract_region(scfg, tail_region_blocks, "tail") def _iter_branch_regions( - bbmap: SCFG, immdoms: Dict[Label, Label], postimmdoms: Dict[Label, Label] + scfg: SCFG, immdoms: Dict[Label, Label], postimmdoms: Dict[Label, Label] ): - for begin, node in [i for i in bbmap.graph.items()]: + for begin, node in [i for i in scfg.graph.items()]: if len(node.jump_targets) > 1: # found branch if begin in postimmdoms: @@ -428,48 +429,48 @@ def _imm_doms(doms: Dict[Label, Set[Label]]) -> Dict[Label, Label]: return out -def _doms(bbmap: SCFG): +def _doms(scfg: SCFG): # compute dom entries = set() preds_table = defaultdict(set) succs_table = defaultdict(set) node: BasicBlock - for src, node in bbmap.graph.items(): + for src, node in scfg.graph.items(): for dst in node.jump_targets: # check dst is in subgraph - if dst in bbmap.graph: + if dst in scfg.graph: preds_table[dst].add(src) succs_table[src].add(dst) - for k in bbmap.graph: + for k in scfg.graph: if not preds_table[k]: entries.add(k) return _find_dominators_internal( - entries, list(bbmap.graph.keys()), preds_table, succs_table + entries, list(scfg.graph.keys()), preds_table, succs_table ) -def _post_doms(bbmap: SCFG): +def _post_doms(scfg: SCFG): # compute post dom entries = set() - for k, v in bbmap.graph.items(): - targets = set(v.jump_targets) & set(bbmap.graph) + for k, v in scfg.graph.items(): + targets = set(v.jump_targets) & set(scfg.graph) if not targets: entries.add(k) preds_table = defaultdict(set) succs_table = defaultdict(set) node: BasicBlock - for src, node in bbmap.graph.items(): + for src, node in scfg.graph.items(): for dst in node.jump_targets: # check dst is in subgraph - if dst in bbmap.graph: + if dst in scfg.graph: preds_table[src].add(dst) succs_table[dst].add(src) return _find_dominators_internal( - entries, list(bbmap.graph.keys()), preds_table, succs_table + entries, list(scfg.graph.keys()), preds_table, succs_table ) @@ -575,18 +576,18 @@ def insert_block_and_control_blocks( scfg.add_block(new_block) -def join_returns(scfg): +def join_returns(scfg: SCFG): """Close the CFG. A closed CFG is a CFG with a unique entry and exit node that have no predescessors and no successors respectively. """ # for all nodes that contain a return - return_nodes = [node for node in scfg.blocks if scfg.blocks[node].is_exiting] + return_nodes = [node for node in scfg.blocks.keys() if scfg.is_exiting(node)] # close if more than one is found if len(return_nodes) > 1: return_solo_label = SyntheticReturn() - scfg.insert_block(return_solo_label, return_nodes, tuple()) + scfg.insert_block(return_nodes, tuple(), block_label=return_solo_label) def join_tails_and_exits(scfg, tails: Set[BlockName], exits: Set[BlockName]): diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 56395b9..742bd64 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -10,6 +10,7 @@ Label, PythonBytecodeLabel, ControlLabel, + BlockName ) from numba_rvsdg.core.datastructures.byte_flow import ByteFlow import dis @@ -17,124 +18,110 @@ class ByteFlowRenderer(object): - def __init__(self): + def __init__(self, byte_flow: ByteFlow): from graphviz import Digraph self.g = Digraph() + self.byte_flow = byte_flow + self.scfg = byte_flow.scfg + self.bcmap_from_bytecode(byte_flow.bc) - def render_region_block( - self, digraph: "Digraph", label: Label - ): + self.render_blocks() + self.render_edges() + self.render_regions() + + def render_regions(self): # render subgraph - graph = regionblock.get_full_graph() - with digraph.subgraph(name=f"cluster_{label}") as subg: - color = "blue" - if regionblock.kind == "branch": - color = "green" - if regionblock.kind == "tail": - color = "purple" - if regionblock.kind == "head": - color = "red" - subg.attr(color=color, label=regionblock.kind) - for label, block in graph.items(): - self.render_block(subg, label, block) - # render edges within this region - self.render_edges(graph) - - def render_basic_block(self, digraph: "Digraph", label: Label, block: BasicBlock): - if isinstance(label, PythonBytecodeLabel): + # graph = regionblock.get_full_graph() + # with digraph.subgraph(name=f"cluster_{label}") as subg: + # color = "blue" + # if regionblock.kind == "branch": + # color = "green" + # if regionblock.kind == "tail": + # color = "purple" + # if regionblock.kind == "head": + # color = "red" + # subg.attr(color=color, label=regionblock.kind) + # for label, block in graph.items(): + # self.render_block(subg, label, block) + # # render edges within this region + # self.render_edges(graph) + pass + + def render_basic_block(self, block_name: BlockName): + block = self.scfg[block_name] + + if isinstance(block.label, PythonBytecodeLabel): instlist = block.get_instructions(self.bcmap) - body = label.__class__.__name__ + ": " + str(label.index) + "\l" + body = str(block_name) + "\l" body += "\l".join( [f"{inst.offset:3}: {inst.opname}" for inst in instlist] + [""] ) - elif isinstance(label, ControlLabel): - body = label.__class__.__name__ + ": " + str(label.index) + elif isinstance(block, ControlLabel): + body = str(block_name) else: - raise Exception("Unknown label type: " + label) - digraph.node(str(label), shape="rect", label=body) - - def render_control_variable_block( - self, digraph: "Digraph", label: Label, block: BasicBlock - ): - if isinstance(label, ControlLabel): - body = label.__class__.__name__ + ": " + str(label.index) + "\l" + raise Exception("Unknown label type: " + block.label) + self.g.node(str(block_name), shape="rect", label=body) + + def render_control_variable_block(self, block_name: BlockName): + block = self.scfg[block_name] + + if isinstance(block.label, ControlLabel): + body = str(block_name) + "\l" body += "\l".join( (f"{k} = {v}" for k, v in block.variable_assignment.items()) ) else: - raise Exception("Unknown label type: " + label) - digraph.node(str(label), shape="rect", label=body) + raise Exception("Unknown label type: " + block.label) + self.g.node(str(block_name), shape="rect", label=body) - def render_branching_block( - self, digraph: "Digraph", label: Label, block: BasicBlock - ): - if isinstance(label, ControlLabel): + def render_branching_block(self, block_name: BlockName): + block = self.scfg[block_name] + if isinstance(block.label, ControlLabel): def find_index(v): if hasattr(v, "offset"): return v.offset if hasattr(v, "index"): return v.index - body = label.__class__.__name__ + ": " + str(label.index) + "\l" + body = str(block_name) + "\l" body += f"variable: {block.variable}\l" body += "\l".join( (f"{k}=>{find_index(v)}" for k, v in block.branch_value_table.items()) ) else: - raise Exception("Unknown label type: " + label) - digraph.node(str(label), shape="rect", label=body) - - def render_block(self, digraph: "Digraph", label: Label, block: BasicBlock): - if type(block) == BasicBlock: - self.render_basic_block(digraph, label, block) - elif type(block) == PythonBytecodeBlock: - self.render_basic_block(digraph, label, block) - elif type(block) == ControlVariableBlock: - self.render_control_variable_block(digraph, label, block) - elif type(block) == BranchBlock: - self.render_branching_block(digraph, label, block) - elif type(block) == RegionBlock: - self.render_region_block(digraph, label, block) - else: - raise Exception("unreachable") - - def render_edges(self, blocks: Dict[Label, BasicBlock]): - for label, block in blocks.items(): - for dst in block.jump_targets: - if dst in blocks: - if type(block) in ( - PythonBytecodeBlock, - BasicBlock, - ControlVariableBlock, - BranchBlock, - ): - self.g.edge(str(label), str(dst)) - elif type(block) == RegionBlock: - if block.exit is not None: - self.g.edge(str(block.exit), str(dst)) - else: - self.g.edge(str(label), str(dst)) - else: - raise Exception("unreachable") - for dst in block.backedges: - # assert dst in blocks - self.g.edge( - str(label), str(dst), style="dashed", color="grey", constraint="0" - ) - - def render_byteflow(self, byteflow: ByteFlow): - self.bcmap_from_bytecode(byteflow.bc) - - # render nodes - for label, block in byteflow.bbmap.graph.items(): - self.render_block(self.g, label, block) - self.render_edges(byteflow.bbmap.graph) - return self.g + raise Exception("Unknown label type: " + block.label) + self.g.node(str(block_name), shape="rect", label=body) + + def render_blocks(self): + for block_name, block in self.scfg.blocks.items(): + if type(block) == BasicBlock: + self.render_basic_block(block_name) + elif type(block) == PythonBytecodeBlock: + self.render_basic_block(block_name) + elif type(block) == ControlVariableBlock: + self.render_control_variable_block(block_name) + elif type(block) == BranchBlock: + self.render_branching_block(block_name) + else: + raise Exception("unreachable") + + def render_edges(self): + + for block_name, out_edges in self.scfg.out_edges.items(): + for out_edge in out_edges: + self.g.edge(str(block_name), str(out_edge)) + + for block_name, back_edges in self.scfg.back_edges.items(): + for back_edge in back_edges: + self.g.edge(str(block_name), str(back_edge), style="dashed", color="grey", constraint="0") def bcmap_from_bytecode(self, bc: dis.Bytecode): - self.bcmap: Dict[int, dis.Instruction] = SCFG.bcmap_from_bytecode(bc) + self.bcmap: Dict[int, dis.Instruction] = ByteFlow.bcmap_from_bytecode(bc) + + def view(self, *args): + self.g.view(*args) logging.basicConfig(level=logging.DEBUG) @@ -142,13 +129,13 @@ def bcmap_from_bytecode(self, bc: dis.Bytecode): def render_func(func): flow = ByteFlow.from_bytecode(func) - ByteFlowRenderer().render_byteflow(flow).view("before") + ByteFlowRenderer(flow).view("before") - cflow = flow._join_returns() - ByteFlowRenderer().render_byteflow(cflow).view("closed") + flow._join_returns() + ByteFlowRenderer(flow).view("closed") - lflow = cflow._restructure_loop() - ByteFlowRenderer().render_byteflow(lflow).view("loop restructured") + flow._restructure_loop() + ByteFlowRenderer(flow).view("loop restructured") - bflow = lflow._restructure_branch() - ByteFlowRenderer().render_byteflow(bflow).view("branch restructured") + flow._restructure_branch() + ByteFlowRenderer(flow).view("branch restructured") diff --git a/numba_rvsdg/tests/simulator.py b/numba_rvsdg/tests/simulator.py index 1cc6ab2..cafa4bf 100644 --- a/numba_rvsdg/tests/simulator.py +++ b/numba_rvsdg/tests/simulator.py @@ -5,7 +5,6 @@ from numba_rvsdg.core.datastructures.basic_block import ( BasicBlock, PythonBytecodeBlock, - RegionBlock, ) from numba_rvsdg.core.datastructures.labels import ( Label, @@ -17,6 +16,7 @@ SyntheticHead, SyntheticTail, SyntheticReturn, + BlockName ) import builtins @@ -50,10 +50,8 @@ class Simulator: Control variable map stack: List[Instruction] Instruction stack - region_stack: List[RegionBlocks] - Stack to hold the recusion level for regions - trace: List[Tuple(label, block)] - List of label, block combinations visisted + trace: List[Tuple(name, block)] + List of names, block combinations visisted branch: Boolean Flag to be set during execution. return_value: Any @@ -64,19 +62,19 @@ class Simulator: def __init__(self, flow: ByteFlow, globals: dict): self.flow = flow + self.scfg = flow.scfg self.globals = ChainMap(globals, builtins.__dict__) self.bcmap = {inst.offset: inst for inst in flow.bc} self.varmap = dict() self.ctrl_varmap = dict() self.stack = [] - self.region_stack = [] self.trace = [] self.branch = None self.return_value = None - def get_block(self, label:Label): - """Return the BasicBlock object for a give label. + def get_block(self, name:BlockName): + """Return the BasicBlock object for a give name. This method is aware of the recusion level of the `Simulator` into the `region_stack`. That is to say, if we have recursed into regions, the @@ -87,8 +85,8 @@ def get_block(self, label:Label): Parameters ---------- - label: Label - The label for which to fetch the BasicBlock + name: BlockName + The name for which to fetch the BasicBlock Return ------ @@ -96,12 +94,7 @@ def get_block(self, label:Label): The requested block """ - # Recursed into regions, return block from region - if self.region_stack: - return self.region_stack[-1].subregion[label] - # Not recursed into regions, return block from ByteFlow - else: - return self.flow.bbmap[label] + return self.flow.scfg[name] def run(self, args): """Run the given simulator with given args. @@ -118,20 +111,20 @@ def run(self, args): """ self.varmap.update(args) - label = PythonBytecodeLabel(index=0) + name = self.flow.scfg.find_head() while True: - action = self.run_BasicBlock(label) + action = self.run_BasicBlock(name) if "return" in action: return action["return"] - label = action["jumpto"] + name = action["jumpto"] - def run_BasicBlock(self, label: Label): + def run_BasicBlock(self, name: BlockName): """Run a BasicBlock. Paramters --------- - label: Label - The Label of the BasicBlock + name: BlockName + The BlockName of the BasicBlock Returns ------- @@ -140,104 +133,51 @@ def run_BasicBlock(self, label: Label): BasicBlock. """ - print("AT", label) - block = self.get_block(label) - self.trace.append((label, block)) - if isinstance(block, RegionBlock): - return self.run_RegionBlock(label) - - if isinstance(label, ControlLabel): - self.run_synth_block(label) - elif isinstance(label, PythonBytecodeLabel): - self.run_PythonBytecodeBlock(label) - if block.fallthrough: - [label] = block.jump_targets - return {"jumpto": label} - elif len(block._jump_targets) == 2: - [br_false, br_true] = block._jump_targets + print("AT", name) + block = self.get_block(name) + self.trace.append((name, block)) + + if isinstance(block.label, ControlLabel): + self.run_synth_block(name) + elif isinstance(block.label, PythonBytecodeLabel): + self.run_PythonBytecodeBlock(name) + if len(self.scfg.out_edges[name]) == 1: + [name] = self.scfg.out_edges[name] + return {"jumpto": name} + elif len(self.scfg.out_edges[name]) == 2: + [br_false, br_true] = self.scfg.out_edges[name] return {"jumpto": br_true if self.branch else br_false} else: return {"return": self.return_value} - def run_RegionBlock(self, label: Label): - """Run region. - - Execute all BasicBlocks in this region. Stay within the region, only - return the action when we jump out of the region or when we return from - within the region. - - Special attention is directed at the use of the `region_stack` here. - Since the blocks for the subregion are stored in the `region.subregion` - graph, we need to use a region aware `get_blocks` in methods such as - `run_BasicBlock` so that we get the correct `BasicBlock`. The net effect - of placing the `region` onto the `region_stack` is that `run_BasicBlock` - will be able to fetch the correct label from the `region.subregion` - graph, and thus be able to run the correct sequence of blocks. - - Parameters - ---------- - label: Label - The Label for the RegionBlock - - Returns - ------- - action: Dict[Str: Int or Boolean or Any] - The action to be taken as a result of having executed the - BasicBlock. - - """ - # Get the RegionBlock and place it onto the region_stack - region: RegionBlock = self.get_block(label) - self.region_stack.append(region) - while True: - # Execute the first block of the region. - action = self.run_BasicBlock(label) - # If we need to return, break and do so - if "return" in action: - break # break and return action - elif "jumpto" in action: - label = action["jumpto"] - # Otherwise check if we stay in the region and break otherwise - if label in region.subregion.graph: - continue # stay in the region - else: - break # break and return action - else: - assert False, "unreachable" # in case of coding errors - # Pop the region from the region stack again and return the final - # action for this region - popped = self.region_stack.pop() - assert(popped == region) - return action - - def run_PythonBytecodeBlock(self, label: PythonBytecodeLabel): + def run_PythonBytecodeBlock(self, name: BlockName): """Run PythonBytecodeBlock Parameters ---------- - label: PythonBytecodeLabel - The Label for the block. + name: BlockName + The BlockName for the block. """ - block: PythonBytecodeBlock = self.get_block(label) + block: PythonBytecodeBlock = self.get_block(name) assert type(block) is PythonBytecodeBlock for inst in block.get_instructions(self.bcmap): self.run_inst(inst) - def run_synth_block(self, label: ControlLabel): + def run_synth_block(self, name: BlockName): """Run a SyntheticBlock Paramaters ---------- - label: ControlLabel - The Label for the block. + name: BlockName + The BlockName for the block. """ - print("----", label) + print("----", name) print(f"control variable map: {self.ctrl_varmap}") - block = self.get_block(label) - handler = getattr(self, f"synth_{type(label).__name__}") - handler(label, block) + block = self.get_block(name) + handler = getattr(self, f"synth_{type(name).__name__}") + handler(name, block) def run_inst(self, inst: Instruction): """Run a bytecode Instruction diff --git a/numba_rvsdg/tests/test_byteflow.py b/numba_rvsdg/tests/test_byteflow.py index 47bc430..da4dedb 100644 --- a/numba_rvsdg/tests/test_byteflow.py +++ b/numba_rvsdg/tests/test_byteflow.py @@ -263,15 +263,15 @@ class TestByteFlow(unittest.TestCase): def test_constructor(self): byteflow = ByteFlow([], []) self.assertEqual(len(byteflow.bc), 0) - self.assertEqual(len(byteflow.bbmap), 0) + self.assertEqual(len(byteflow.scfg), 0) def test_from_bytecode(self): - bbmap = SCFG() + scfg = SCFG() - bbmap.add_block('python_bytecode', begin=0, end=10, label=get_label_class('python_bytecode')()) - expected = ByteFlow(bc=bytecode, bbmap=bbmap) + scfg.add_block('python_bytecode', begin=0, end=10, label=get_label_class('python_bytecode')()) + expected = ByteFlow(bc=bytecode, scfg=scfg) received = ByteFlow.from_bytecode(fun) - self.assertEqual(expected.bbmap, received.bbmap) + self.assertEqual(expected.scfg, received.scfg) if __name__ == "__main__": diff --git a/numba_rvsdg/tests/test_fig3.py b/numba_rvsdg/tests/test_fig3.py index e3e2525..bdf4d3f 100644 --- a/numba_rvsdg/tests/test_fig3.py +++ b/numba_rvsdg/tests/test_fig3.py @@ -29,5 +29,5 @@ def make_flow(): dis.Instruction("RETURN_VALUE", 1, None, None, "", 20, None, False), ] flow = FlowInfo.from_bytecode(bc) - bbmap = flow.build_basicblocks() - return ByteFlow(bc=bc, bbmap=bbmap) + scfg = flow.build_basicblocks() + return ByteFlow(bc=bc, scfg=scfg) diff --git a/numba_rvsdg/tests/test_fig4.py b/numba_rvsdg/tests/test_fig4.py index d30636b..605efcc 100644 --- a/numba_rvsdg/tests/test_fig4.py +++ b/numba_rvsdg/tests/test_fig4.py @@ -28,5 +28,5 @@ def make_flow(): dis.Instruction("RETURN_VALUE", 1, None, None, "", 18, None, False), ] flow = FlowInfo.from_bytecode(bc) - bbmap = flow.build_basicblocks() - return ByteFlow(bc=bc, bbmap=bbmap) + scfg = flow.build_basicblocks() + return ByteFlow(bc=bc, scfg=scfg) diff --git a/numba_rvsdg/tests/test_simulate.py b/numba_rvsdg/tests/test_simulate.py index f97a37b..9b87479 100644 --- a/numba_rvsdg/tests/test_simulate.py +++ b/numba_rvsdg/tests/test_simulate.py @@ -2,29 +2,6 @@ from numba_rvsdg.tests.simulator import Simulator import unittest -# flow = ByteFlow.from_bytecode(foo) -# #pprint(flow.bbmap) -# flow = flow.restructure() -# #pprint(flow.bbmap) -# # pprint(rtsflow.bbmap) -# ByteFlowRenderer().render_byteflow(flow).view() -# print(dis(foo)) -# -# sim = Simulator(flow, foo.__globals__) -# ret = sim.run(dict(x=1)) -# assert ret == foo(x=1) -# -# #sim = Simulator(flow, foo.__globals__) -# #ret = sim.run(dict(x=100)) -# #assert ret == foo(x=100) - -# You can use the following snipppet to visually debug the restructured -# byteflow: -# -# ByteFlowRenderer().render_byteflow(flow).view() -# -# - class SimulatorTest(unittest.TestCase): def _run(self, func, flow, kwargs): @@ -42,7 +19,7 @@ def foo(x): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # if case self._run(foo, flow, {"x": 1}) @@ -57,7 +34,7 @@ def foo(x): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # loop bypass case self._run(foo, flow, {"x": 0}) @@ -76,7 +53,7 @@ def foo(x): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # loop bypass case self._run(foo, flow, {"x": 0}) @@ -95,7 +72,7 @@ def foo(x): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # loop bypass case self._run(foo, flow, {"x": 0}) @@ -119,7 +96,7 @@ def foo(x): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # no loop self._run(foo, flow, {"x": 0}) @@ -143,7 +120,7 @@ def foo(x): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # loop bypass self._run(foo, flow, {"x": 0}) @@ -159,7 +136,7 @@ def foo(x, y): return (x > 0 and x < 10) or (y > 0 and y < 10) flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() self._run(foo, flow, {"x": 5, "y": 5}) @@ -173,7 +150,7 @@ def foo(s, e): return c flow = ByteFlow.from_bytecode(foo) - flow = flow.restructure() + # flow = flow.restructure() # no looping self._run(foo, flow, {"s": 0, "e": 0}) From e03688e094abbb442517046e6ff6a57b7c1738fe Mon Sep 17 00:00:00 2001 From: kc611 Date: Mon, 27 Mar 2023 16:15:49 +0530 Subject: [PATCH 03/30] Refactored joins transformations and respective tests --- .../core/datastructures/basic_block.py | 12 +- numba_rvsdg/core/datastructures/byte_flow.py | 7 +- numba_rvsdg/core/datastructures/flow_info.py | 10 +- numba_rvsdg/core/datastructures/labels.py | 25 +- numba_rvsdg/core/datastructures/region.py | 6 +- numba_rvsdg/core/datastructures/scfg.py | 85 +- numba_rvsdg/core/transformations.py | 149 ++-- numba_rvsdg/rendering/rendering.py | 15 +- numba_rvsdg/tests/simulator.py | 98 +- numba_rvsdg/tests/test_byteflow.py | 46 +- numba_rvsdg/tests/test_scfg.py | 112 +-- numba_rvsdg/tests/test_transforms.py | 840 +++++++++++------- numba_rvsdg/tests/test_utils.py | 32 +- 13 files changed, 827 insertions(+), 610 deletions(-) diff --git a/numba_rvsdg/core/datastructures/basic_block.py b/numba_rvsdg/core/datastructures/basic_block.py index 3660fcc..45d87f5 100644 --- a/numba_rvsdg/core/datastructures/basic_block.py +++ b/numba_rvsdg/core/datastructures/basic_block.py @@ -21,7 +21,7 @@ class BasicBlock: def __post_init__(self, name_gen): block_name = name_gen.new_block_name(label=self.label) - object.__setattr__(self, 'block_name', block_name) + object.__setattr__(self, "block_name", block_name) @dataclass(frozen=True) @@ -63,12 +63,13 @@ class BranchBlock(BasicBlock): variable: str = None branch_value_table: dict = None + # Maybe we can register new blocks over here instead of static lists block_types = { - 'basic': BasicBlock, - 'python_bytecode': PythonBytecodeBlock, - 'control_variable': ControlVariableBlock, - 'branch': BranchBlock + "basic": BasicBlock, + "python_bytecode": PythonBytecodeBlock, + "control_variable": ControlVariableBlock, + "branch": BranchBlock, } @@ -77,4 +78,3 @@ def get_block_class(block_type_string): return block_types[block_type_string] else: raise TypeError(f"Block Type {block_type_string} not recognized.") - diff --git a/numba_rvsdg/core/datastructures/byte_flow.py b/numba_rvsdg/core/datastructures/byte_flow.py index 412729f..8274abe 100644 --- a/numba_rvsdg/core/datastructures/byte_flow.py +++ b/numba_rvsdg/core/datastructures/byte_flow.py @@ -5,7 +5,11 @@ from numba_rvsdg.core.datastructures.flow_info import FlowInfo from numba_rvsdg.core.utils import _logger, _LogWrap -from numba_rvsdg.core.transformations import restructure_loop, restructure_branch, join_returns +from numba_rvsdg.core.transformations import ( + restructure_loop, + restructure_branch, + join_returns, +) @dataclass(frozen=True) @@ -42,4 +46,3 @@ def restructure(self): @staticmethod def bcmap_from_bytecode(bc: dis.Bytecode): return {inst.offset: inst for inst in bc} - diff --git a/numba_rvsdg/core/datastructures/flow_info.py b/numba_rvsdg/core/datastructures/flow_info.py index 7ba8730..a37a180 100644 --- a/numba_rvsdg/core/datastructures/flow_info.py +++ b/numba_rvsdg/core/datastructures/flow_info.py @@ -77,10 +77,12 @@ def build_basicblocks(self: "FlowInfo", end_offset=None) -> "SCFG": end_offset = _next_inst_offset(self.last_offset) for begin, end in zip(offsets, [*offsets[1:], end_offset]): - names[begin] = scfg.add_block(block_type='python_bytecode', - block_label=PythonBytecodeLabel(), - begin=begin, - end=end) + names[begin] = scfg.add_block( + block_type="python_bytecode", + block_label=PythonBytecodeLabel(), + begin=begin, + end=end, + ) for begin, end in zip(offsets, [*offsets[1:], end_offset]): targets: Tuple[BlockName, ...] diff --git a/numba_rvsdg/core/datastructures/labels.py b/numba_rvsdg/core/datastructures/labels.py index 4db2e3f..a23debd 100644 --- a/numba_rvsdg/core/datastructures/labels.py +++ b/numba_rvsdg/core/datastructures/labels.py @@ -61,17 +61,17 @@ class SynthenticAssignment(ControlLabel): # Maybe we can register new labels over here instead of static lists label_types = { - 'label': Label, - 'python_bytecode': PythonBytecodeLabel, - 'control': ControlLabel, - 'synth_branch': SyntheticBranch, - 'synth_tail': SyntheticTail, - 'synth_exit': SyntheticExit, - 'synth_head': SyntheticHead, - 'synth_return': SyntheticReturn, - 'synth_latch': SyntheticLatch, - 'synth_exit_latch': SyntheticExitingLatch, - 'synth_assign': SynthenticAssignment + "label": Label, + "python_bytecode": PythonBytecodeLabel, + "control": ControlLabel, + "synth_branch": SyntheticBranch, + "synth_tail": SyntheticTail, + "synth_exit": SyntheticExit, + "synth_head": SyntheticHead, + "synth_return": SyntheticReturn, + "synth_latch": SyntheticLatch, + "synth_exit_latch": SyntheticExitingLatch, + "synth_assign": SynthenticAssignment, } @@ -82,17 +82,18 @@ def get_label_class(label_type_string): raise TypeError(f"Block Type {label_type_string} not recognized.") - @dataclass(frozen=True, order=True) class BlockName: name: str ... + @dataclass(frozen=True, order=True) class RegionName: name: str ... + @dataclass class NameGenerator: index: int = 0 diff --git a/numba_rvsdg/core/datastructures/region.py b/numba_rvsdg/core/datastructures/region.py index 147ed34..5c4c0c8 100644 --- a/numba_rvsdg/core/datastructures/region.py +++ b/numba_rvsdg/core/datastructures/region.py @@ -1,4 +1,3 @@ - from dataclasses import dataclass, field, InitVar from numba_rvsdg.core.datastructures.labels import NameGenerator, RegionName, BlockName @@ -10,7 +9,7 @@ class Region: """Region Name Generator associated with this Region. Note: This is an initialization only argument and not a class attribute.""" - + region_name: RegionName = field(init=False) """Unique name identifier for this region""" @@ -20,5 +19,4 @@ class Region: def __post_init__(self, name_gen): region_name = name_gen.new_region_name(kind=self.kind) - object.__setattr__(self, 'region_name', region_name) - + object.__setattr__(self, "region_name", region_name) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 3d09834..77a7318 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -1,4 +1,3 @@ - import yaml import itertools @@ -6,37 +5,34 @@ from typing import Set, Tuple, Dict, List, Iterator from dataclasses import dataclass, field -from numba_rvsdg.core.datastructures.basic_block import ( - BasicBlock, - get_block_class -) +from numba_rvsdg.core.datastructures.basic_block import BasicBlock, get_block_class from numba_rvsdg.core.datastructures.region import Region from numba_rvsdg.core.datastructures.labels import ( Label, BlockName, NameGenerator, RegionName, - get_label_class + get_label_class, ) @dataclass(frozen=True) class SCFG: """Maps of BlockNames to respective BasicBlocks. - And stores the jump targets and back edges for - blocks within the graph.""" + And stores the jump targets and back edges for + blocks within the graph.""" blocks: Dict[BlockName, BasicBlock] = field(default_factory=dict) out_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) back_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) - in_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) # Design question + in_edges: Dict[BlockName, Set[BlockName]] = field( + default_factory=dict + ) # Design question regions: Dict[RegionName, Region] = field(default_factory=dict) - name_gen: NameGenerator = field( - default_factory=NameGenerator, compare=False - ) + name_gen: NameGenerator = field(default_factory=NameGenerator, compare=False) def __getitem__(self, index): return self.blocks[index] @@ -110,10 +106,11 @@ def compute_scc_subgraph(self, subgraph) -> List[Set[BlockName]]: Strongly-connected component for detecting loops inside a subgraph. """ from numba_rvsdg.networkx_vendored.scc import scc + out_edges = self.out_edges class GraphWrap: - def __init__(self, graph, subgraph): + def __init__(self, graph: Dict[BlockName, BasicBlock], subgraph): self.graph = graph self.subgraph = subgraph @@ -144,7 +141,9 @@ def find_headers_and_entries( headers: Set[BlockName] = set() for outside in self.exclude_blocks(subgraph): - nodes_jump_in_loop = subgraph.intersection(self.out_edges[self.blocks[outside]]) + nodes_jump_in_loop = subgraph.intersection( + self.out_edges[self.blocks[outside]] + ) headers.update(nodes_jump_in_loop) if nodes_jump_in_loop: entries.add(outside) @@ -175,7 +174,7 @@ def find_exiting_and_exits( exiting.add(inside) exits.add(jt) # any returns - if self.blocks[inside].is_exiting: + if self.is_exiting(self.blocks[inside]): exiting.add(inside) return exiting, exits @@ -216,15 +215,22 @@ def check_graph(self): # del self.in_edges[name] # self.check_graph() - def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], block_type: str = 'basic', block_label: Label = Label(), **block_args): + def insert_block( + self, + predecessors: Set[BlockName], + successors: Set[BlockName], + block_type: str = "basic", + block_label: Label = Label(), + **block_args, + ): # TODO: needs a diagram and documentaion # initialize new block - block_type = get_block_class(block_type) - new_block_name = self.add_block(block_type, **block_args) + new_block_name = self.add_block( + block_type, block_label=block_label, **block_args + ) # Replace any arcs from any of predecessors to any of successors with # an arc through the inserted block instead. - for pred_name in predecessors: # For every predecessor # Remove all successors from its out edges @@ -233,7 +239,7 @@ def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], self.out_edges[pred_name].add(new_block_name) # For inserted block, the predecessor in an in-edge self.in_edges[new_block_name].add(pred_name) - + for success_name in successors: # For every sucessor # Remove all predecessors from it's in edges @@ -241,18 +247,16 @@ def insert_block(self, predecessors: Set[BlockName], successors: Set[BlockName], # Add the inserted block as in edge self.in_edges[success_name].add(new_block_name) # For inserted block, the sucessor in an out-edge - self.out_edges[new_block_name].add(success_name) + self.out_edges[new_block_name].add(success_name) self.check_graph() return new_block_name - def add_block(self, block_type: str = 'basic', block_label: Label = Label(), **block_args): + def add_block( + self, block_type: str = "basic", block_label: Label = Label(), **block_args + ) -> BlockName: block_type = get_block_class(block_type) - new_block = block_type( - **block_args, - label=block_label, - name_gen=self.name_gen - ) + new_block = block_type(**block_args, label=block_label, name_gen=self.name_gen) name = new_block.block_name self.blocks[name] = new_block @@ -272,24 +276,23 @@ def add_connections(self, block_name, out_edges=(), back_edges=()): self.check_graph() - @staticmethod def from_yaml(yaml_string): data = yaml.safe_load(yaml_string) return SCFG.from_dict(data) @staticmethod - def from_dict(graph_dict): + def from_dict(graph_dict: Dict[str, Dict]): scfg = SCFG() ref_dict = {} for block_ref, block_attrs in graph_dict.items(): - block_class = get_block_class(block_attrs['type']) - block_args = {} - label_class = get_label_class(block_attrs.get('label_type', 'label')) - label_info = block_attrs.get('label_info', None) - block_args = {'label': label_class(label_info)} - block_name = scfg.add_block(block_class, **block_args) + block_class = block_attrs["type"] + block_args = block_attrs.get("block_args", {}) + label_class = get_label_class(block_attrs.get("label_type", "label")) + label_info = block_attrs.get("label_info", None) + block_label = label_class(label_info) + block_name = scfg.add_block(block_class, block_label, **block_args) ref_dict[block_ref] = block_name for block_ref, block_attrs in graph_dict.items(): @@ -300,7 +303,7 @@ def from_dict(graph_dict): out_edges = set(ref_dict[out_ref] for out_ref in out_refs) back_edges = set(ref_dict[back_ref] for back_ref in back_refs) scfg.add_connections(block_name, out_edges, back_edges) - + scfg.check_graph() return scfg, ref_dict @@ -310,15 +313,15 @@ def to_yaml(self): for key, value in self.blocks.items(): out_edges = [f"{i}" for i in self.out_edges[key]] - out_edges = str(out_edges).replace("\'", "\"") + out_edges = str(out_edges).replace("'", '"') back_edges = [f"{i}" for i in self.back_edges[key]] - jump_target_str= f""" + jump_target_str = f""" "{str(key)}": type: {type(value)} out: {out_edges}""" if back_edges: - back_edges = str(back_edges).replace("\'", "\"") + back_edges = str(back_edges).replace("'", '"') jump_target_str += f""" back: {back_edges}""" yaml_string += dedent(jump_target_str) @@ -326,9 +329,9 @@ def to_yaml(self): return yaml_string # def to_dict(self): - # block_map_graph = self.graph + # scfg_graph = self.graph # graph_dict = {} - # for key, value in block_map_graph.items(): + # for key, value in scfg_graph.items(): # curr_dict = {} # curr_dict["jt"] = [f"{i.index}" for i in value._out_edges] # if value.backedges: diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 83e6f40..1f49f26 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -8,9 +8,10 @@ SyntheticExitingLatch, SyntheticExit, SyntheticReturn, + SyntheticTail, SynthenticAssignment, PythonBytecodeLabel, - BlockName + BlockName, ) from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.basic_block import ( @@ -22,7 +23,7 @@ from numba_rvsdg.core.utils import _logger -def loop_restructure_helper(scfg: SCFG, loop: Set[Label]): +def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): """Loop Restructuring Applies the algorithm LOOP RESTRUCTURING from section 4.1 of Bahmann2015. @@ -33,14 +34,14 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[Label]): ---------- scfg: SCFG The SCFG containing the loop - loop: Set[Label] + loop: Set[BlockName] The loop (strongly connected components) that is to be restructured """ headers, entries = scfg.find_headers_and_entries(loop) exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(loop) - #assert len(entries) == 1 + # assert len(entries) == 1 headers_were_unified = False # If there are multiple headers, insert assignment and control blocks, @@ -48,7 +49,7 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[Label]): if len(headers) > 1: headers_were_unified = True solo_head_label = SyntheticHead(str(scfg.clg.new_index())) - scfg.insert_block_and_control_blocks(solo_head_label, entries, headers) + insert_block_and_control_blocks(solo_head_label, entries, headers) loop.add(solo_head_label) loop_head: Label = solo_head_label else: @@ -59,8 +60,11 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[Label]): backedge_blocks = [ block for block in loop if headers.intersection(scfg[block].jump_targets) ] - if (len(backedge_blocks) == 1 and len(exiting_blocks) == 1 - and backedge_blocks[0] == next(iter(exiting_blocks))): + if ( + len(backedge_blocks) == 1 + and len(exiting_blocks) == 1 + and backedge_blocks[0] == next(iter(exiting_blocks)) + ): scfg.add_block(scfg.graph.pop(backedge_blocks[0]).replace_backedge(loop_head)) return @@ -87,9 +91,13 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[Label]): # depending on the state of the CFG and what is needed exit_value_table = dict(((i, j) for i, j in enumerate(exit_blocks))) if needs_synth_exit: - backedge_value_table = dict((i, j) for i, j in enumerate((loop_head, synth_exit))) + backedge_value_table = dict( + (i, j) for i, j in enumerate((loop_head, synth_exit)) + ) else: - backedge_value_table = dict((i, j) for i, j in enumerate((loop_head, next(iter(exit_blocks))))) + backedge_value_table = dict( + (i, j) for i, j in enumerate((loop_head, next(iter(exit_blocks)))) + ) if headers_were_unified: header_value_table = scfg[solo_head_label].branch_value_table else: @@ -127,9 +135,13 @@ def reverse_lookup(d, value): # Setup the variables in the assignment table to point to # the correct blocks if needs_synth_exit: - variable_assignment[exit_variable] = reverse_lookup(exit_value_table, jt) - variable_assignment[backedge_variable] = reverse_lookup(backedge_value_table, - synth_exit if needs_synth_exit else next(iter(exit_blocks))) + variable_assignment[exit_variable] = reverse_lookup( + exit_value_table, jt + ) + variable_assignment[backedge_variable] = reverse_lookup( + backedge_value_table, + synth_exit if needs_synth_exit else next(iter(exit_blocks)), + ) # Create the actual control variable block synth_assign_block = ControlVariableBlock( label=synth_assign, @@ -151,9 +163,13 @@ def reverse_lookup(d, value): # Setup the variables in the assignment table to point to # the correct blocks variable_assignment = {} - variable_assignment[backedge_variable] = reverse_lookup(backedge_value_table, loop_head) + variable_assignment[backedge_variable] = reverse_lookup( + backedge_value_table, loop_head + ) if needs_synth_exit: - variable_assignment[exit_variable] = reverse_lookup(header_value_table, jt) + variable_assignment[exit_variable] = reverse_lookup( + header_value_table, jt + ) # Update the backedge block - remove any existing backedges # that point to the headers, no need to add a backedge, # since it will be contained in the SyntheticExitingLatch @@ -186,7 +202,10 @@ def reverse_lookup(d, value): # Insert the exiting latch, add it to the loop and to the graph. synth_exiting_latch_block = BranchBlock( label=synth_exiting_latch, - _jump_targets=(synth_exit if needs_synth_exit else next(iter(exit_blocks)), loop_head), + _jump_targets=( + synth_exit if needs_synth_exit else next(iter(exit_blocks)), + loop_head, + ), backedges=(loop_head,), variable=backedge_variable, branch_value_table=backedge_value_table, @@ -281,9 +300,7 @@ def _find_branch_regions(scfg: SCFG, begin: Label, end: Label) -> Set[Label]: return branch_regions -def find_tail_blocks( - scfg: SCFG, begin: Set[Label], head_region_blocks, branch_regions -): +def find_tail_blocks(scfg: SCFG, begin: Set[Label], head_region_blocks, branch_regions): tail_subregion = set((b for b in scfg.graph.keys())) tail_subregion.difference_update(head_region_blocks) for reg in branch_regions: @@ -377,7 +394,9 @@ def restructure_branch(scfg: SCFG): else: # Insert SyntheticBranch tail_headers, _ = scfg.find_headers_and_entries(tail_region_blocks) - synthetic_branch_block_label = SyntheticBranch(str(scfg.name_gen.new_index())) + synthetic_branch_block_label = SyntheticBranch( + str(scfg.name_gen.new_index()) + ) scfg.insert_block(synthetic_branch_block_label, (begin,), tail_headers) # Recompute regions. @@ -454,7 +473,7 @@ def _doms(scfg: SCFG): def _post_doms(scfg: SCFG): # compute post dom entries = set() - for k, v in scfg.graph.items(): + for k, v in scfg.blocks.items(): targets = set(v.jump_targets) & set(scfg.graph) if not targets: entries.add(k) @@ -462,7 +481,7 @@ def _post_doms(scfg: SCFG): succs_table = defaultdict(set) node: BasicBlock - for src, node in scfg.graph.items(): + for src, node in scfg.blocks.items(): for dst in node.jump_targets: # check dst is in subgraph if dst in scfg.graph: @@ -470,7 +489,7 @@ def _post_doms(scfg: SCFG): succs_table[dst].add(src) return _find_dominators_internal( - entries, list(scfg.graph.keys()), preds_table, succs_table + entries, list(scfg.blocks.keys()), preds_table, succs_table ) @@ -521,59 +540,62 @@ def _find_dominators_internal(entries, nodes, preds_table, succs_table): def insert_block_and_control_blocks( - scfg, label: Label, predecessors: Set[BlockName], successors: Set[BlockName] -): + scfg: SCFG, + predecessors: Set[BlockName], + successors: Set[BlockName], + block_type: str = "basic", + block_label: Label = Label(), + **block_args +): # TODO: needs a diagram and documentaion # name of the variable for this branching assignment - branch_variable = scfg.name_gen.new_name(label) - # initialize new block, which will hold the branching table - new_block = BranchBlock( - label=label, - _jump_targets=tuple(successors), - backedges=set(), - variable=branch_variable, - branch_value_table=branch_value_table, - name_gen=scfg.name_gen - ) + branch_variable = scfg.name_gen.new_var_name() # initial value of the assignment branch_variable_value = 0 # store for the mapping from variable value to blockname branch_value_table = {} + # initialize new block, which will hold the branching table + branch_block_name = scfg.add_block( + "branch", + block_label, + branch_variable=branch_variable, + branch_value_table=branch_value_table, + ) + scfg.add_connections(branch_block_name, tuple(successors), set()) + # Replace any arcs from any of predecessors to any of successors with # an arc through the to be inserted block instead. - for name in predecessors: - block = scfg.graph[name] - jt = list(block.out_edges) + + for pred_name in predecessors: + pred_outs = scfg.out_edges[pred_name] # Need to create synthetic assignments for each arc from a # predecessors to a successor and insert it between the predecessor # and the newly created block - for s in set(jt).intersection(successors): + for s in pred_outs.intersection(successors): synth_assign = SynthenticAssignment() variable_assignment = {} variable_assignment[branch_variable] = branch_variable_value - synth_assign_block = ControlVariableBlock( - label=synth_assign, - _jump_targets=(branch_variable,), - backedges=(), + + # add block + control_block_name = scfg.add_block( + "control_variable", + synth_assign, variable_assignment=variable_assignment, - name_gen=scfg.name_gen ) - # add block - scfg.add_block(synth_assign_block) + scfg.add_connections(control_block_name, set((branch_block_name,)), set()) # update branching table branch_value_table[branch_variable_value] = s # update branching variable branch_variable_value += 1 - # replace previous successor with synth_assign - jt[jt.index(s)] = synth_assign - # finally, replace the out_edges - scfg.add_block( - scfg.graph.pop(name).replace_jump_targets(out_edges=tuple(jt)) - ) - assert new_block.block_name == branch_variable - # add block to scfg - scfg.add_block(new_block) + # Rewrite old edges + scfg.out_edges[pred_name].remove(s) + scfg.out_edges[pred_name].add(control_block_name) + + scfg.in_edges[s].remove(pred_name) + scfg.in_edges[s].add(control_block_name) + + return branch_block_name def join_returns(scfg: SCFG): @@ -587,10 +609,10 @@ def join_returns(scfg: SCFG): # close if more than one is found if len(return_nodes) > 1: return_solo_label = SyntheticReturn() - scfg.insert_block(return_nodes, tuple(), block_label=return_solo_label) + scfg.insert_block(set(return_nodes), set(), block_label=return_solo_label) -def join_tails_and_exits(scfg, tails: Set[BlockName], exits: Set[BlockName]): +def join_tails_and_exits(scfg: SCFG, tails: Set[BlockName], exits: Set[BlockName]): if len(tails) == 1 and len(exits) == 1: # no-op solo_tail_name = next(iter(tails)) @@ -601,25 +623,22 @@ def join_tails_and_exits(scfg, tails: Set[BlockName], exits: Set[BlockName]): # join only exits solo_tail_name = next(iter(tails)) solo_exit_label = SyntheticExit() - exit_block = scfg.insert_block(solo_exit_label, tails, exits) - solo_exit_name = exit_block.block_name + solo_exit_name = scfg.insert_block(tails, exits, block_label=solo_exit_label) return solo_tail_name, solo_exit_name if len(tails) >= 2 and len(exits) == 1: # join only tails solo_tail_label = SyntheticTail() solo_exit_name = next(iter(exits)) - tail_block = scfg.insert_block(solo_tail_label, tails, exits) - solo_tail_name = tail_block.block_name + solo_tail_name = scfg.insert_block(tails, exits, block_label=solo_tail_label) return solo_tail_name, solo_exit_name if len(tails) >= 2 and len(exits) >= 2: # join both tails and exits solo_tail_label = SyntheticTail() solo_exit_label = SyntheticExit() - tail_block = scfg.insert_block(solo_tail_label, tails, exits) - solo_tail_name = tail_block.block_name - exit_block = scfg.insert_block(solo_exit_label, set((solo_tail_label,)), exits) - solo_exit_name = exit_block.block_name + solo_tail_name = scfg.insert_block(tails, exits, block_label=solo_tail_label) + solo_exit_name = scfg.insert_block( + set((solo_tail_name,)), exits, block_label=solo_exit_label + ) return solo_tail_name, solo_exit_name - diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 742bd64..3619974 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -10,7 +10,7 @@ Label, PythonBytecodeLabel, ControlLabel, - BlockName + BlockName, ) from numba_rvsdg.core.datastructures.byte_flow import ByteFlow import dis @@ -65,7 +65,7 @@ def render_basic_block(self, block_name: BlockName): def render_control_variable_block(self, block_name: BlockName): block = self.scfg[block_name] - + if isinstance(block.label, ControlLabel): body = str(block_name) + "\l" body += "\l".join( @@ -79,6 +79,7 @@ def render_branching_block(self, block_name: BlockName): block = self.scfg[block_name] if isinstance(block.label, ControlLabel): + def find_index(v): if hasattr(v, "offset"): return v.offset @@ -108,14 +109,20 @@ def render_blocks(self): raise Exception("unreachable") def render_edges(self): - + for block_name, out_edges in self.scfg.out_edges.items(): for out_edge in out_edges: self.g.edge(str(block_name), str(out_edge)) for block_name, back_edges in self.scfg.back_edges.items(): for back_edge in back_edges: - self.g.edge(str(block_name), str(back_edge), style="dashed", color="grey", constraint="0") + self.g.edge( + str(block_name), + str(back_edge), + style="dashed", + color="grey", + constraint="0", + ) def bcmap_from_bytecode(self, bc: dis.Bytecode): self.bcmap: Dict[int, dis.Instruction] = ByteFlow.bcmap_from_bytecode(bc) diff --git a/numba_rvsdg/tests/simulator.py b/numba_rvsdg/tests/simulator.py index cafa4bf..7588411 100644 --- a/numba_rvsdg/tests/simulator.py +++ b/numba_rvsdg/tests/simulator.py @@ -5,6 +5,8 @@ from numba_rvsdg.core.datastructures.basic_block import ( BasicBlock, PythonBytecodeBlock, + ControlVariableBlock, + BranchBlock, ) from numba_rvsdg.core.datastructures.labels import ( Label, @@ -16,7 +18,7 @@ SyntheticHead, SyntheticTail, SyntheticReturn, - BlockName + BlockName, ) import builtins @@ -73,7 +75,7 @@ def __init__(self, flow: ByteFlow, globals: dict): self.branch = None self.return_value = None - def get_block(self, name:BlockName): + def get_block(self, name: BlockName): """Return the BasicBlock object for a give name. This method is aware of the recusion level of the `Simulator` into the @@ -197,44 +199,58 @@ def run_inst(self, inst: Instruction): print(f"stack after: {self.stack}") ### Synthetic Instructions ### - def synth_SynthenticAssignment(self, control_label, block): + def synth_SynthenticAssignment( + self, control_label: BlockName, block: ControlVariableBlock + ): self.ctrl_varmap.update(block.variable_assignment) - def _synth_branch(self, control_label, block): + def _synth_branch(self, control_label: BlockName, block: BranchBlock): jump_target = block.branch_value_table[self.ctrl_varmap[block.variable]] self.branch = bool(block._jump_targets.index(jump_target)) - def synth_SyntheticExitingLatch(self, control_label, block): + def synth_SyntheticExitingLatch( + self, control_label: BlockName, block: ControlVariableBlock + ): self._synth_branch(control_label, block) - def synth_SyntheticHead(self, control_label, block): + def synth_SyntheticHead( + self, control_label: BlockName, block: ControlVariableBlock + ): self._synth_branch(control_label, block) - def synth_SyntheticExit(self, control_label, block): + def synth_SyntheticExit( + self, control_label: BlockName, block: ControlVariableBlock + ): self._synth_branch(control_label, block) - def synth_SyntheticReturn(self, control_label, block): + def synth_SyntheticReturn( + self, control_label: BlockName, block: ControlVariableBlock + ): pass - def synth_SyntheticTail(self, control_label, block): + def synth_SyntheticTail( + self, control_label: BlockName, block: ControlVariableBlock + ): pass - def synth_SyntheticBranch(self, control_label, block): + def synth_SyntheticBranch( + self, control_label: BlockName, block: ControlVariableBlock + ): pass ### Bytecode Instructions ### - def op_LOAD_CONST(self, inst): + def op_LOAD_CONST(self, inst: Instruction): self.stack.append(inst.argval) - def op_COMPARE_OP(self, inst): + def op_COMPARE_OP(self, inst: Instruction): arg1 = self.stack.pop() arg2 = self.stack.pop() self.stack.append(eval(f"{arg2} {inst.argval} {arg1}")) - def op_LOAD_FAST(self, inst): + def op_LOAD_FAST(self, inst: Instruction): self.stack.append(self.varmap[inst.argval]) - def op_LOAD_GLOBAL(self, inst): + def op_LOAD_GLOBAL(self, inst: Instruction): v = self.globals[inst.argval] if inst.argrepr.startswith("NULL"): append_null = True @@ -243,22 +259,22 @@ def op_LOAD_GLOBAL(self, inst): else: raise NotImplementedError - def op_STORE_FAST(self, inst): + def op_STORE_FAST(self, inst: Instruction): val = self.stack.pop() self.varmap[inst.argval] = val - def op_CALL_FUNCTION(self, inst): + def op_CALL_FUNCTION(self, inst: Instruction): args = [self.stack.pop() for _ in range(inst.argval)][::-1] fn = self.stack.pop() res = fn(*args) self.stack.append(res) - def op_GET_ITER(self, inst): + def op_GET_ITER(self, inst: Instruction): val = self.stack.pop() res = iter(val) self.stack.append(res) - def op_FOR_ITER(self, inst): + def op_FOR_ITER(self, inst: Instruction): tos = self.stack[-1] try: ind = next(tos) @@ -269,58 +285,58 @@ def op_FOR_ITER(self, inst): self.branch = False self.stack.append(ind) - def op_INPLACE_ADD(self, inst): + def op_INPLACE_ADD(self, inst: Instruction): rhs = self.stack.pop() lhs = self.stack.pop() lhs += rhs self.stack.append(lhs) - def op_RETURN_VALUE(self, inst): + def op_RETURN_VALUE(self, inst: Instruction): v = self.stack.pop() self.return_value = v - def op_JUMP_ABSOLUTE(self, inst): + def op_JUMP_ABSOLUTE(self, inst: Instruction): pass - def op_JUMP_FORWARD(self, inst): + def op_JUMP_FORWARD(self, inst: Instruction): pass - def op_POP_JUMP_IF_FALSE(self, inst): + def op_POP_JUMP_IF_FALSE(self, inst: Instruction): self.branch = not self.stack.pop() - def op_POP_JUMP_IF_TRUE(self, inst): + def op_POP_JUMP_IF_TRUE(self, inst: Instruction): self.branch = bool(self.stack.pop()) - def op_JUMP_IF_TRUE_OR_POP(self, inst): + def op_JUMP_IF_TRUE_OR_POP(self, inst: Instruction): if self.stack[-1]: self.branch = True else: self.stack.pop() self.branch = False - def op_JUMP_IF_FALSE_OR_POP(self, inst): + def op_JUMP_IF_FALSE_OR_POP(self, inst: Instruction): if not self.stack[-1]: self.branch = True else: self.stack.pop() self.branch = False - def op_POP_TOP(self, inst): + def op_POP_TOP(self, inst: Instruction): self.stack.pop() - def op_RESUME(self, inst): + def op_RESUME(self, inst: Instruction): pass - def op_PRECALL(self, inst): + def op_PRECALL(self, inst: Instruction): pass - def op_CALL_FUNCTION(self, inst): + def op_CALL_FUNCTION(self, inst: Instruction): args = [self.stack.pop() for _ in range(inst.argval)][::-1] fn = self.stack.pop() res = fn(*args) self.stack.append(res) - def op_CALL(self, inst): + def op_CALL(self, inst: Instruction): args = [self.stack.pop() for _ in range(inst.argval)][::-1] first, second = self.stack.pop(), self.stack.pop() if first == None: @@ -330,42 +346,42 @@ def op_CALL(self, inst): res = func(*args) self.stack.append(res) - def op_BINARY_OP(self, inst): + def op_BINARY_OP(self, inst: Instruction): rhs, lhs, op = self.stack.pop(), self.stack.pop(), inst.argrepr op = op if len(op) == 1 else op[0] self.stack.append(eval(f"{lhs} {op} {rhs}")) - def op_JUMP_BACKWARD(self, inst): + def op_JUMP_BACKWARD(self, inst: Instruction): pass - def op_POP_JUMP_FORWARD_IF_TRUE(self, inst): + def op_POP_JUMP_FORWARD_IF_TRUE(self, inst: Instruction): self.branch = self.stack[-1] self.stack.pop() - def op_POP_JUMP_BACKWARD_IF_TRUE(self, inst): + def op_POP_JUMP_BACKWARD_IF_TRUE(self, inst: Instruction): self.branch = self.stack[-1] self.stack.pop() - def op_POP_JUMP_FORWARD_IF_FALSE(self, inst): + def op_POP_JUMP_FORWARD_IF_FALSE(self, inst: Instruction): self.branch = not self.stack[-1] self.stack.pop() - def op_POP_JUMP_BACKWARD_IF_FALSE(self, inst): + def op_POP_JUMP_BACKWARD_IF_FALSE(self, inst: Instruction): self.branch = not self.stack[-1] self.stack.pop() - def op_POP_JUMP_FORWARD_IF_NOT_NONE(self, inst): + def op_POP_JUMP_FORWARD_IF_NOT_NONE(self, inst: Instruction): self.branch = self.stack[-1] is not None self.stack.pop() - def op_POP_JUMP_BACKWARD_IF_NOT_NONE(self, inst): + def op_POP_JUMP_BACKWARD_IF_NOT_NONE(self, inst: Instruction): self.branch = self.stack[-1] is not None self.stack.pop() - def op_POP_JUMP_FORWARD_IF_NONE(self, inst): + def op_POP_JUMP_FORWARD_IF_NONE(self, inst: Instruction): self.branch = self.stack[-1] is None self.stack.pop() - def op_POP_JUMP_BACKWARD_IF_NONE(self, inst): + def op_POP_JUMP_BACKWARD_IF_NONE(self, inst: Instruction): self.branch = self.stack[-1] is None self.stack.pop() diff --git a/numba_rvsdg/tests/test_byteflow.py b/numba_rvsdg/tests/test_byteflow.py index da4dedb..9f0030f 100644 --- a/numba_rvsdg/tests/test_byteflow.py +++ b/numba_rvsdg/tests/test_byteflow.py @@ -2,7 +2,12 @@ import unittest from numba_rvsdg.core.datastructures.basic_block import PythonBytecodeBlock -from numba_rvsdg.core.datastructures.labels import PythonBytecodeLabel, NameGenerator, BlockName, get_label_class +from numba_rvsdg.core.datastructures.labels import ( + PythonBytecodeLabel, + NameGenerator, + BlockName, + get_label_class, +) from numba_rvsdg.core.datastructures.byte_flow import ByteFlow from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.flow_info import FlowInfo @@ -115,7 +120,7 @@ def test_constructor(self): end=8, _jump_targets=(), backedges=(), - name_gen=name_gen + name_gen=name_gen, ) self.assertEqual(block.label, PythonBytecodeLabel()) self.assertEqual(block.begin, 0) @@ -125,23 +130,23 @@ def test_constructor(self): self.assertEqual(block.jump_targets, ()) self.assertEqual(block.backedges, ()) - block_name = BlockName('pythonbytecodelabel_0') + block_name = BlockName("pythonbytecodelabel_0") self.assertEqual(block.block_name, block_name) def test_is_jump_target(self): name_gen = NameGenerator(index=0) label = PythonBytecodeLabel() - + block = PythonBytecodeBlock( label=label, begin=0, end=8, _jump_targets=(name_gen.new_block_name(label),), backedges=(), - name_gen=name_gen + name_gen=name_gen, ) - self.assertEqual(block.block_name, BlockName(name='pythonbytecodelabel_1')) - self.assertEqual(block.jump_targets, (BlockName(name='pythonbytecodelabel_0'),)) + self.assertEqual(block.block_name, BlockName(name="pythonbytecodelabel_1")) + self.assertEqual(block.jump_targets, (BlockName(name="pythonbytecodelabel_0"),)) self.assertFalse(block.is_exiting) def test_get_instructions(self): @@ -154,7 +159,7 @@ def test_get_instructions(self): end=8, _jump_targets=(), backedges=(), - name_gen=name_gen + name_gen=name_gen, ) expected = [ Instruction( @@ -242,17 +247,15 @@ def test_from_bytecode(self): def test_build_basic_blocks(self): name_gen = NameGenerator(index=0) - expected = SCFG( - graph={} - ) + expected = SCFG(graph={}) block = PythonBytecodeBlock( - label=PythonBytecodeLabel(), - begin=0, - end=10, - _jump_targets=(), - backedges=(), - name_gen=name_gen - ) + label=PythonBytecodeLabel(), + begin=0, + end=10, + _jump_targets=(), + backedges=(), + name_gen=name_gen, + ) expected.add_block(block) received = FlowInfo.from_bytecode(bytecode).build_basicblocks() @@ -268,7 +271,12 @@ def test_constructor(self): def test_from_bytecode(self): scfg = SCFG() - scfg.add_block('python_bytecode', begin=0, end=10, label=get_label_class('python_bytecode')()) + scfg.add_block( + "python_bytecode", + begin=0, + end=10, + label=get_label_class("python_bytecode")(), + ) expected = ByteFlow(bc=bytecode, scfg=scfg) received = ByteFlow.from_bytecode(fun) self.assertEqual(expected.scfg, received.scfg) diff --git a/numba_rvsdg/tests/test_scfg.py b/numba_rvsdg/tests/test_scfg.py index 37cb46c..bc2abaf 100644 --- a/numba_rvsdg/tests/test_scfg.py +++ b/numba_rvsdg/tests/test_scfg.py @@ -1,4 +1,3 @@ - from unittest import main from textwrap import dedent from numba_rvsdg.core.datastructures.scfg import SCFG @@ -8,11 +7,11 @@ from numba_rvsdg.core.datastructures.labels import Label, NameGenerator -class TestBlockMapConversion(MapComparator): - +class TestscfgConversion(MapComparator): def test_yaml_conversion(self): # Case # 1: Acyclic graph, no back-edges - cases = [""" + cases = [ + """ "0": type: "basic" out: ["1", "2"] @@ -28,7 +27,7 @@ def test_yaml_conversion(self): "4": type: "basic" out: []""", - # Case # 2: Cyclic graph, no back edges + # Case # 2: Cyclic graph, no back edges """ "0": type: "basic" @@ -48,7 +47,7 @@ def test_yaml_conversion(self): "5": type: "basic" out: ["3", "4"]""", - # Case # 3: Graph with backedges + # Case # 3: Graph with backedges """ "0": type: "basic" @@ -65,79 +64,50 @@ def test_yaml_conversion(self): "4": type: "basic" out: ["2", "3"] - back: ["2"]"""] + back: ["2"]""", + ] for case in cases: case = dedent(case) - block_map, ref_dict = SCFG.from_yaml(case) + scfg, ref_dict = SCFG.from_yaml(case) # TODO: use ref_dict for comparision - self.assertEqual(case, block_map.to_yaml()) + self.assertEqual(case, scfg.to_yaml()) def test_dict_conversion(self): # Case # 1: Acyclic graph, no back-edges - cases = [{ - "0": - {"type": "basic", - "out": ["1", "2"]}, - "1": - {"type": "basic", - "out": ["3"]}, - "2": - {"type": "basic", - "out": ["4"]}, - "3": - {"type": "basic", - "out": ["4"]}, - "4": - {"type": "basic", - "out": []}}, - # Case # 2: Cyclic graph, no back edges - { - "0": - {"type": "basic", - "out": ["1", "2"]}, - "1": - {"type": "basic", - "out": ["5"]}, - "2": - {"type": "basic", - "out": ["1", "5"]}, - "3": - {"type": "basic", - "out": ["0"]}, - "4": - {"type": "basic", - "out": []}, - "5": - {"type": "basic", - "out": ["3", "4"]}}, - # Case # 3: Graph with backedges - { - "0": - {"type": "basic", - "out": ["1"]}, - "1": - {"type": "basic", - "out": ["2", "3"]}, - "2": - {"type": "basic", - "out": ["4"]}, - "3": - {"type": "basic", - "out": []}, - "4": - {"type": "basic", - "out": ["2", "3"], - "back": ["2"]} - }] + cases = [ + { + "0": {"type": "basic", "out": ["1", "2"]}, + "1": {"type": "basic", "out": ["3"]}, + "2": {"type": "basic", "out": ["4"]}, + "3": {"type": "basic", "out": ["4"]}, + "4": {"type": "basic", "out": []}, + }, + # Case # 2: Cyclic graph, no back edges + { + "0": {"type": "basic", "out": ["1", "2"]}, + "1": {"type": "basic", "out": ["5"]}, + "2": {"type": "basic", "out": ["1", "5"]}, + "3": {"type": "basic", "out": ["0"]}, + "4": {"type": "basic", "out": []}, + "5": {"type": "basic", "out": ["3", "4"]}, + }, + # Case # 3: Graph with backedges + { + "0": {"type": "basic", "out": ["1"]}, + "1": {"type": "basic", "out": ["2", "3"]}, + "2": {"type": "basic", "out": ["4"]}, + "3": {"type": "basic", "out": []}, + "4": {"type": "basic", "out": ["2", "3"], "back": ["2"]}, + }, + ] for case in cases: - block_map = SCFG.from_dict(case) - self.assertEqual(case, block_map.to_dict()) + scfg = SCFG.from_dict(case) + self.assertEqual(case, scfg.to_dict()) class TestSCFGIterator(MapComparator): - def test_scfg_iter(self): name_generator = NameGenerator() block_0 = BasicBlock(name_generator, Label()) @@ -146,16 +116,18 @@ def test_scfg_iter(self): (block_0.block_name, block_0), (block_1.block_name, block_1), ] - scfg = SCFG.from_yaml(""" + scfg = SCFG.from_yaml( + """ "0": type: "basic" out: ["1"] "1": type: "basic" - """) + """ + ) received = list(scfg) self.assertEqual(expected, received) + if __name__ == "__main__": main() - diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index 7ff7d48..f579cdc 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -1,4 +1,3 @@ - from unittest import main from numba_rvsdg.core.datastructures.labels import ( @@ -7,466 +6,585 @@ SyntheticExit, ) from numba_rvsdg.core.datastructures.scfg import SCFG -from numba_rvsdg.core.transformations import loop_restructure_helper -from numba_rvsdg.tests.test_utils import MapComparator +from numba_rvsdg.core.transformations import ( + loop_restructure_helper, + join_returns, + join_tails_and_exits, +) +from numba_rvsdg.tests.test_utils import SCFGComparator -class TestInsertBlock(MapComparator): +class TestInsertBlock(SCFGComparator): def test_linear(self): original = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["2"] + type: "basic" + out: ["2"] "1": - jt: [] + type: "basic" + out: [] "2": - jt: ["1"] + type: "basic" + out: ["1"] """ - expected_block_map = SCFG.from_yaml(expected) - original_block_map.insert_block( - ControlLabel("2"), wrap_id(("0",)), wrap_id(("1",)) - ) - self.assertMapEqual(expected_scfg, original_scfg) + expected_scfg, _ = SCFG.from_yaml(expected) + + preds = set((block_ref_orig["0"],)) + succs = set((block_ref_orig["1"],)) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) def test_dual_predecessor(self): original = """ "0": - jt: ["2"] + type: "basic" + out: ["2"] "1": - jt: ["2"] + type: "basic" + out: ["2"] "2": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["3"] + type: "basic" + out: ["3"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: [] + type: "basic" + out: [] "3": - jt: ["2"] + type: "basic" + out: ["2"] """ - expected_block_map = SCFG.from_yaml(expected) - original_block_map.insert_block( - ControlLabel("3"), wrap_id(("0", "1")), wrap_id(("2",)) - ) - self.assertMapEqual(expected_scfg, original_scfg) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = set((block_ref_orig["0"], block_ref_orig["1"])) + succs = set((block_ref_orig["2"],)) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) def test_dual_successor(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: [] + type: "basic" + out: [] "2": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["3"] + type: "basic" + out: ["3"] "1": - jt: [] + type: "basic" + out: [] "2": - jt: [] + type: "basic" + out: [] "3": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] """ - expected_block_map = SCFG.from_yaml(expected) - original_block_map.insert_block( - ControlLabel("3"), - wrap_id(("0",)), - wrap_id(("1", "2")), - ) - self.assertMapEqual(expected_scfg, original_scfg) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = set((block_ref_orig["0"],)) + succs = set((block_ref_orig["1"], block_ref_orig["2"])) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) def test_dual_predecessor_and_dual_successor(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: [] + type: "basic" + out: [] "4": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["5"] + type: "basic" + out: ["5"] "2": - jt: ["5"] + type: "basic" + out: ["5"] "3": - jt: [] + type: "basic" + out: [] "4": - jt: [] + type: "basic" + out: [] "5": - jt: ["3", "4"] + type: "basic" + out: ["3", "4"] """ - expected_block_map = SCFG.from_yaml(expected) - original_block_map.insert_block( - ControlLabel("5"), - wrap_id(("1", "2")), - wrap_id(("3", "4")), - ) - self.assertMapEqual(expected_scfg, original_scfg) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = set((block_ref_orig["1"], block_ref_orig["2"])) + succs = set((block_ref_orig["3"], block_ref_orig["4"])) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["1", "4"] + type: "basic" + out: ["1", "4"] "3": - jt: ["0"] + type: "basic" + out: ["0"] "4": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["5"] + type: "basic" + out: ["5"] "2": - jt: ["1", "5"] + type: "basic" + out: ["1", "5"] "3": - jt: ["0"] + type: "basic" + out: ["0"] "4": - jt: [] + type: "basic" + out: [] "5": - jt: ["3", "4"] + type: "basic" + out: ["3", "4"] """ - expected_block_map = SCFG.from_yaml(expected) - original_block_map.insert_block( - ControlLabel("5"), - wrap_id(("1", "2")), - wrap_id(("3", "4")), - ) - self.assertMapEqual(expected_scfg, original_scfg) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = set((block_ref_orig["1"], block_ref_orig["2"])) + succs = set((block_ref_orig["3"], block_ref_orig["4"])) + original_scfg.insert_block(preds, succs) + self.assertSCFGEqual(expected_scfg, original_scfg) -class TestJoinReturns(MapComparator): + +class TestJoinReturns(SCFGComparator): def test_two_returns(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: [] + type: "basic" + out: [] "2": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["3"] + type: "basic" + out: ["3"] "3": - jt: [] + type: "basic" + label_type: "synth_return" + out: [] """ - expected_block_map = SCFG.from_yaml(expected) - original_block_map.join_returns() - self.assertMapEqual(expected_block_map, original_block_map) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + join_returns(original_scfg) + + self.assertSCFGEqual(expected_scfg, original_scfg) -class TestJoinTailsAndExits(MapComparator): +class TestJoinTailsAndExits(SCFGComparator): def test_join_tails_and_exits_case_00(self): original = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: [] + type: "basic" + out: [] """ - expected_block_map = SCFG.from_yaml(expected) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = wrap_id(("0",)) - exits = wrap_id(("1",)) - solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( - tails, exits - ) + tails = set((block_ref_orig["0"],)) + exits = set((block_ref_orig["1"],)) + join_tails_and_exits(original_scfg, tails, exits) - self.assertMapEqual(expected_scfg, original_scfg) - self.assertEqual(ControlLabel("0"), solo_tail_label) - self.assertEqual(ControlLabel("1"), solo_exit_label) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_join_tails_and_exits_case_01(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["3"] + type: "basic" + out: ["3"] "3": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["4"] + type: "basic" + out: ["4"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["3"] + type: "basic" + out: ["3"] "3": - jt: [] + type: "basic" + out: [] "4": - jt: ["1", "2"] + type: "basic" + label_type: "synth_exit" + out: ["1", "2"] """ - expected_block_map = SCFG.from_yaml(expected) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = wrap_id(("0",)) - exits = wrap_id(("1", "2")) - solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( - tails, exits - ) + tails = set((block_ref_orig["0"],)) + exits = set((block_ref_orig["1"], block_ref_orig["2"])) + join_tails_and_exits(original_scfg, tails, exits) - self.assertMapEqual(expected_scfg, original_scfg) - self.assertEqual(ControlLabel("0"), solo_tail_label) - self.assertEqual(SyntheticExit("4"), solo_exit_label) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_join_tails_and_exits_case_02_01(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["3"] + type: "basic" + out: ["3"] "3": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["4"] + type: "basic" + out: ["4"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: [] + type: "basic" + out: [] "4": - jt: ["3"] + type: "basic" + label_type: "synth_tail" + out: ["3"] """ - expected_block_map = SCFG.from_yaml(expected) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = wrap_id(("1", "2")) - exits = wrap_id(("3",)) - solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( - tails, exits - ) + tails = set((block_ref_orig["1"], block_ref_orig["2"])) + exits = set((block_ref_orig["3"],)) + join_tails_and_exits(original_scfg, tails, exits) - self.assertMapEqual(expected_scfg, original_scfg) - self.assertEqual(SyntheticTail("4"), solo_tail_label) - self.assertEqual(ControlLabel("3"), solo_exit_label) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_join_tails_and_exits_case_02_02(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["1", "3"] + type: "basic" + out: ["1", "3"] "3": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["4"] + type: "basic" + out: ["4"] "2": - jt: ["1", "4"] + type: "basic" + out: ["1", "4"] "3": - jt: [] + type: "basic" + out: [] "4": - jt: ["3"] + type: "basic" + label_type: "synth_tail" + out: ["3"] """ - expected_block_map = SCFG.from_yaml(expected) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = wrap_id(("1", "2")) - exits = wrap_id(("3",)) + tails = set((block_ref_orig["1"], block_ref_orig["2"])) + exits = set((block_ref_orig["3"],)) - solo_tail_label, solo_exit_label = original_scfg.join_tails_and_exits( - tails, exits - ) - self.assertMapEqual(expected_scfg, original_scfg) - self.assertEqual(SyntheticTail("4"), solo_tail_label) - self.assertEqual(ControlLabel("3"), solo_exit_label) + join_tails_and_exits(original_scfg, tails, exits) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_join_tails_and_exits_case_03_01(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: ["5"] + type: "basic" + out: ["5"] "4": - jt: ["5"] + type: "basic" + out: ["5"] "5": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["6"] + type: "basic" + out: ["6"] "2": - jt: ["6"] + type: "basic" + out: ["6"] "3": - jt: ["5"] + type: "basic" + out: ["5"] "4": - jt: ["5"] + type: "basic" + out: ["5"] "5": - jt: [] + type: "basic" + out: [] "6": - jt: ["7"] + type: "basic" + label_type: "synth_tail" + out: ["7"] "7": - jt: ["3", "4"] + type: "basic" + label_type: "synth_exit" + out: ["3", "4"] """ - expected_block_map = SCFG.from_yaml(expected) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = wrap_id(("1", "2")) - exits = wrap_id(("3", "4")) - solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( - tails, exits - ) - self.assertMapEqual(expected_scfg, original_scfg) - self.assertEqual(SyntheticTail("6"), solo_tail_label) - self.assertEqual(SyntheticExit("7"), solo_exit_label) + tails = set((block_ref_orig["1"], block_ref_orig["2"])) + exits = set((block_ref_orig["3"], block_ref_orig["4"])) + join_tails_and_exits(original_scfg, tails, exits) + + self.assertSCFGEqual(expected_scfg, original_scfg) def test_join_tails_and_exits_case_03_02(self): original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["1", "4"] + type: "basic" + out: ["1", "4"] "3": - jt: ["5"] + type: "basic" + out: ["5"] "4": - jt: ["5"] + type: "basic" + out: ["5"] "5": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) + original_scfg, block_ref_orig = SCFG.from_yaml(original) expected = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["6"] + type: "basic" + out: ["6"] "2": - jt: ["1", "6"] + type: "basic" + out: ["1", "6"] "3": - jt: ["5"] + type: "basic" + out: ["5"] "4": - jt: ["5"] + type: "basic" + out: ["5"] "5": - jt: [] + type: "basic" + out: [] "6": - jt: ["7"] + type: "basic" + label_type: "synth_tail" + out: ["7"] "7": - jt: ["3", "4"] + type: "basic" + label_type: "synth_exit" + out: ["3", "4"] """ - expected_block_map = SCFG.from_yaml(expected) - tails = wrap_id(("1", "2")) - exits = wrap_id(("3", "4")) - solo_tail_label, solo_exit_label = original_block_map.join_tails_and_exits( - tails, exits - ) - self.assertMapEqual(expected_scfg, original_scfg) - self.assertEqual(SyntheticTail("6"), solo_tail_label) - self.assertEqual(SyntheticExit("7"), solo_exit_label) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + tails = set((block_ref_orig["1"], block_ref_orig["2"])) + exits = set((block_ref_orig["3"], block_ref_orig["4"])) + join_tails_and_exits(original_scfg, tails, exits) + self.assertSCFGEqual(expected_scfg, original_scfg) -class TestLoopRestructure(MapComparator): +class TestLoopRestructure(SCFGComparator): def test_no_op_mono(self): """Loop consists of a single Block.""" original = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "2": - jt: [] + type: "basic" + out: [] """ expected = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] be: ["1"] "2": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) - expected_block_map = SCFG.from_yaml(expected) - loop_restructure_helper(original_block_map, set(wrap_id({"1"}))) - self.assertMapEqual(expected_block_map, original_block_map) + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + loop_restructure_helper(original_scfg, set(wrap_id({"1"}))) + + self.assertSCFGEqual(expected_scfg, original_scfg) def test_no_op(self): """Loop consists of two blocks, but it's in form.""" - original =""" + original = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["2"] + type: "basic" + out: ["2"] "2": - jt: ["1", "3"] + type: "basic" + out: ["1", "3"] "3": - jt: [] + type: "basic" + out: [] """ - expected =""" + expected = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["2"] + type: "basic" + out: ["2"] "2": - jt: ["1", "3"] + type: "basic" + out: ["1", "3"] be: ["1"] "3": - jt: [] + type: "basic" + out: [] """ - original_block_map = SCFG.from_yaml(original) - expected_block_map = SCFG.from_yaml(expected) - loop_restructure_helper(original_block_map, set(wrap_id({"1", "2"}))) - self.assertMapEqual(expected_block_map, original_block_map) + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + loop_restructure_helper(original_scfg, set(wrap_id({"1", "2"}))) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_backedge_not_exiting(self): """Loop has a backedge not coming from the exiting block. @@ -475,35 +593,46 @@ def test_backedge_not_exiting(self): """ original = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["2", "3"] + type: "basic" + out: ["2", "3"] "2": - jt: ["1"] + type: "basic" + out: ["1"] "3": - jt: [] + type: "basic" + out: [] """ expected = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["2", "5"] + type: "basic" + out: ["2", "5"] "2": - jt: ["6"] + type: "basic" + out: ["6"] "3": - jt: [] + type: "basic" + out: [] "4": - jt: ["1", "3"] + type: "basic" + out: ["1", "3"] be: ["1"] "5": - jt: ["4"] + type: "basic" + out: ["4"] "6": - jt: ["4"] + type: "basic" + out: ["4"] """ - original_block_map = SCFG.from_yaml(original) - expected_block_map = SCFG.from_yaml(expected) - loop_restructure_helper(original_block_map, set(wrap_id({"1", "2"}))) - self.assertMapEqual(expected_block_map, original_block_map) + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + loop_restructure_helper(original_scfg, set(wrap_id({"1", "2"}))) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_exit(self): """Loop has two exiting blocks. @@ -513,159 +642,218 @@ def test_double_exit(self): """ original = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["2"] + type: "basic" + out: ["2"] "2": - jt: ["3", "4"] + type: "basic" + out: ["3", "4"] "3": - jt: ["1", "4"] + type: "basic" + out: ["1", "4"] "4": - jt: [] + type: "basic" + out: [] """ expected = """ "0": - jt: ["1"] + type: "basic" + out: ["1"] "1": - jt: ["2"] + type: "basic" + out: ["2"] "2": - jt: ["3", "6"] + type: "basic" + out: ["3", "6"] "3": - jt: ["7", "8"] + type: "basic" + out: ["7", "8"] "4": - jt: [] + type: "basic" + out: [] "5": - jt: ["1", "4"] + type: "basic" + out: ["1", "4"] be: ["1"] "6": - jt: ["5"] + type: "basic" + out: ["5"] "7": - jt: ["5"] + type: "basic" + out: ["5"] "8": - jt: ["5"] + type: "basic" + out: ["5"] """ - original_block_map = SCFG.from_yaml(original) - expected_block_map = SCFG.from_yaml(expected) - loop_restructure_helper(original_block_map, set(wrap_id({"1", "2", "3"}))) - self.assertMapEqual(expected_block_map, original_block_map) + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + loop_restructure_helper(original_scfg, set(wrap_id({"1", "2", "3"}))) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_header(self): - """ This is like the example from Bahman2015 fig. 3 -- + """This is like the example from Bahman2015 fig. 3 -- but with one exiting block removed.""" original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: ["2", "5"] + type: "basic" + out: ["2", "5"] "4": - jt: ["1"] + type: "basic" + out: ["1"] "5": - jt: [] + type: "basic" + out: [] """ expected = """ "0": - jt: ["7", "8"] + type: "basic" + out: ["7", "8"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: ["10", "11"] + type: "basic" + out: ["10", "11"] "4": - jt: ["12"] + type: "basic" + out: ["12"] "5": - jt: [] + type: "basic" + out: [] "6": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "7": - jt: ["6"] + type: "basic" + out: ["6"] "8": - jt: ["6"] + type: "basic" + out: ["6"] "9": - jt: ["5", "6"] + type: "basic" + out: ["5", "6"] be: ["6"] "10": - jt: ["9"] + type: "basic" + out: ["9"] "11": - jt: ["9"] + type: "basic" + out: ["9"] "12": - jt: ["9"] + type: "basic" + out: ["9"] """ - original_block_map = SCFG.from_yaml(original) - expected_block_map = SCFG.from_yaml(expected) - loop_restructure_helper(original_block_map, set(wrap_id({"1", "2", "3", "4"}))) - self.assertMapEqual(expected_block_map, original_block_map) + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + loop_restructure_helper(original_scfg, set(wrap_id({"1", "2", "3", "4"}))) + self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_header_double_exiting(self): - """ This is like the example from Bahman2015 fig. 3. + """This is like the example from Bahman2015 fig. 3. Two headers that need to be multiplexed to, on additional branch that becomes the exiting latch and one branch that becomes the exit. - + """ original = """ "0": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: ["2", "5"] + type: "basic" + out: ["2", "5"] "4": - jt: ["1", "6"] + type: "basic" + out: ["1", "6"] "5": - jt: ["7"] + type: "basic" + out: ["7"] "6": - jt: ["7"] + type: "basic" + out: ["7"] "7": - jt: [] + type: "basic" + out: [] """ expected = """ "0": - jt: ["10", "9"] + type: "basic" + out: ["10", "9"] "1": - jt: ["3"] + type: "basic" + out: ["3"] "2": - jt: ["4"] + type: "basic" + out: ["4"] "3": - jt: ["13", "14"] + type: "basic" + out: ["13", "14"] "4": - jt: ["15", "16"] + type: "basic" + out: ["15", "16"] "5": - jt: ["7"] + type: "basic" + out: ["7"] "6": - jt: ["7"] + type: "basic" + out: ["7"] "7": - jt: [] + type: "basic" + out: [] "8": - jt: ["1", "2"] + type: "basic" + out: ["1", "2"] "9": - jt: ["8"] + type: "basic" + out: ["8"] "10": - jt: ["8"] + type: "basic" + out: ["8"] "11": - jt: ["12", "8"] + type: "basic" + out: ["12", "8"] be: ["8"] "12": - jt: ["5", "6"] + type: "basic" + out: ["5", "6"] "13": - jt: ["11"] + type: "basic" + out: ["11"] "14": - jt: ["11"] + type: "basic" + out: ["11"] "15": - jt: ["11"] + type: "basic" + out: ["11"] "16": - jt: ["11"] + type: "basic" + out: ["11"] """ - original_block_map = SCFG.from_yaml(original) - expected_block_map = SCFG.from_yaml(expected) - loop_restructure_helper(original_block_map, set(wrap_id({"1", "2", "3", "4"}))) - self.assertMapEqual(expected_block_map, original_block_map) + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + loop_restructure_helper(original_scfg, set(wrap_id({"1", "2", "3", "4"}))) + self.assertSCFGEqual(expected_scfg, original_scfg) + if __name__ == "__main__": main() diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index c820e68..3346c02 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -1,21 +1,21 @@ from unittest import TestCase -class MapComparator(TestCase): - def assertMapEqual(self, first_map, second_map): +from numba_rvsdg.core.datastructures.scfg import SCFG + + +class SCFGComparator(TestCase): + def assertSCFGEqual(self, first_scfg: SCFG, second_scfg: SCFG): for key1, key2 in zip( - sorted(first_map.graph.keys(), key=lambda x: x.index), - sorted(second_map.graph.keys(), key=lambda x: x.index), + sorted(first_scfg.blocks.keys(), key=lambda x: x.name), + sorted(second_scfg.blocks.keys(), key=lambda x: x.name), ): - # compare indices of labels - self.assertEqual(key1.index, key2.index) - # compare indices of jump_targets - self.assertEqual( - sorted([j.index for j in first_map[key1]._jump_targets]), - sorted([j.index for j in second_map[key2]._jump_targets]), - ) - # compare indices of backedges - self.assertEqual( - sorted([j.index for j in first_map[key1].backedges]), - sorted([j.index for j in second_map[key2].backedges]), - ) + block_1 = first_scfg[key1] + block_2 = second_scfg[key2] + + # compare labels + self.assertEqual(type(block_1.label), type(block_2.label)) + # compare edges + self.assertEqual(first_scfg.out_edges[key1], second_scfg.out_edges[key2]) + self.assertEqual(first_scfg.in_edges[key1], second_scfg.in_edges[key2]) + self.assertEqual(first_scfg.back_edges[key1], second_scfg.back_edges[key2]) From d9f0b3c165f1c71e4b81bb85134cc1b66696b585 Mon Sep 17 00:00:00 2001 From: kc611 Date: Mon, 27 Mar 2023 19:24:53 +0530 Subject: [PATCH 04/30] Refactored tests and changes edge storage in SCFG to lists --- .../core/datastructures/basic_block.py | 9 +- numba_rvsdg/core/datastructures/flow_info.py | 6 +- numba_rvsdg/core/datastructures/scfg.py | 54 ++-- numba_rvsdg/core/transformations.py | 2 +- numba_rvsdg/tests/test_byteflow.py | 54 +--- numba_rvsdg/tests/test_scfg.py | 211 +++++++++++++++- numba_rvsdg/tests/test_transforms.py | 231 ++---------------- numba_rvsdg/tests/test_utils.py | 15 ++ 8 files changed, 289 insertions(+), 293 deletions(-) diff --git a/numba_rvsdg/core/datastructures/basic_block.py b/numba_rvsdg/core/datastructures/basic_block.py index 45d87f5..6a703a3 100644 --- a/numba_rvsdg/core/datastructures/basic_block.py +++ b/numba_rvsdg/core/datastructures/basic_block.py @@ -73,8 +73,15 @@ class BranchBlock(BasicBlock): } -def get_block_class(block_type_string): +def get_block_class(block_type_string: str): if block_type_string in block_types: return block_types[block_type_string] else: raise TypeError(f"Block Type {block_type_string} not recognized.") + +def get_block_class_str(basic_block: BasicBlock): + for key, value in block_types.items(): + if isinstance(basic_block, value): + return key + else: + raise TypeError(f"Block Type of {basic_block} not recognized.") diff --git a/numba_rvsdg/core/datastructures/flow_info.py b/numba_rvsdg/core/datastructures/flow_info.py index a37a180..528ba62 100644 --- a/numba_rvsdg/core/datastructures/flow_info.py +++ b/numba_rvsdg/core/datastructures/flow_info.py @@ -1,5 +1,5 @@ import dis -from typing import Set, Tuple, Dict, Sequence +from typing import Set, Tuple, Dict, Sequence, List from dataclasses import dataclass, field from numba_rvsdg.core.datastructures.basic_block import PythonBytecodeBlock @@ -85,13 +85,13 @@ def build_basicblocks(self: "FlowInfo", end_offset=None) -> "SCFG": ) for begin, end in zip(offsets, [*offsets[1:], end_offset]): - targets: Tuple[BlockName, ...] + targets: List[BlockName] term_offset = _prev_inst_offset(end) if term_offset not in self.jump_insts: # implicit jump targets = (names[end],) else: - targets = tuple(names[o] for o in self.jump_insts[term_offset]) + targets = [names[o] for o in self.jump_insts[term_offset]] block_name = names[begin] scfg.add_connections(block_name, targets, []) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 77a7318..d107545 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -5,7 +5,7 @@ from typing import Set, Tuple, Dict, List, Iterator from dataclasses import dataclass, field -from numba_rvsdg.core.datastructures.basic_block import BasicBlock, get_block_class +from numba_rvsdg.core.datastructures.basic_block import BasicBlock, get_block_class, get_block_class_str from numba_rvsdg.core.datastructures.region import Region from numba_rvsdg.core.datastructures.labels import ( Label, @@ -24,9 +24,9 @@ class SCFG: blocks: Dict[BlockName, BasicBlock] = field(default_factory=dict) - out_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) - back_edges: Dict[BlockName, Set[BlockName]] = field(default_factory=dict) - in_edges: Dict[BlockName, Set[BlockName]] = field( + out_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict) + back_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict) + in_edges: Dict[BlockName, List[BlockName]] = field( default_factory=dict ) # Design question @@ -217,8 +217,8 @@ def check_graph(self): def insert_block( self, - predecessors: Set[BlockName], - successors: Set[BlockName], + predecessors: List[BlockName], + successors: List[BlockName], block_type: str = "basic", block_label: Label = Label(), **block_args, @@ -234,20 +234,24 @@ def insert_block( for pred_name in predecessors: # For every predecessor # Remove all successors from its out edges - self.out_edges[pred_name].difference_update(successors) + for successor in successors: + if successor in self.out_edges[pred_name]: + self.out_edges[pred_name].remove(successor) # Add the inserted block as out edge - self.out_edges[pred_name].add(new_block_name) + self.out_edges[pred_name].append(new_block_name) # For inserted block, the predecessor in an in-edge - self.in_edges[new_block_name].add(pred_name) + self.in_edges[new_block_name].append(pred_name) for success_name in successors: # For every sucessor # Remove all predecessors from it's in edges - self.in_edges[success_name].difference_update(predecessors) + for pred in predecessors: + if pred in self.in_edges[success_name]: + self.in_edges[success_name].remove(pred) # Add the inserted block as in edge - self.in_edges[success_name].add(new_block_name) + self.in_edges[success_name].append(new_block_name) # For inserted block, the sucessor in an out-edge - self.out_edges[new_block_name].add(success_name) + self.out_edges[new_block_name].append(success_name) self.check_graph() return new_block_name @@ -256,23 +260,23 @@ def add_block( self, block_type: str = "basic", block_label: Label = Label(), **block_args ) -> BlockName: block_type = get_block_class(block_type) - new_block = block_type(**block_args, label=block_label, name_gen=self.name_gen) + new_block: BlockName = block_type(**block_args, label=block_label, name_gen=self.name_gen) name = new_block.block_name self.blocks[name] = new_block - self.in_edges[name] = set() - self.back_edges[name] = set() - self.out_edges[name] = set() + self.in_edges[name] = [] + self.back_edges[name] = [] + self.out_edges[name] = [] return name - def add_connections(self, block_name, out_edges=(), back_edges=()): + def add_connections(self, block_name, out_edges=[], back_edges=[]): self.out_edges[block_name] = out_edges self.back_edges[block_name] = back_edges for edge in itertools.chain(out_edges, back_edges): - self.in_edges[edge].add(block_name) + self.in_edges[edge].append(block_name) self.check_graph() @@ -296,12 +300,12 @@ def from_dict(graph_dict: Dict[str, Dict]): ref_dict[block_ref] = block_name for block_ref, block_attrs in graph_dict.items(): - out_refs = block_attrs.get("out", set()) - back_refs = block_attrs.get("back", set()) + out_refs = block_attrs.get("out", list()) + back_refs = block_attrs.get("back", list()) block_name = ref_dict[block_ref] - out_edges = set(ref_dict[out_ref] for out_ref in out_refs) - back_edges = set(ref_dict[back_ref] for back_ref in back_refs) + out_edges = list(ref_dict[out_ref] for out_ref in out_refs) + back_edges = list(ref_dict[back_ref] for back_ref in back_refs) scfg.add_connections(block_name, out_edges, back_edges) scfg.check_graph() @@ -313,15 +317,15 @@ def to_yaml(self): for key, value in self.blocks.items(): out_edges = [f"{i}" for i in self.out_edges[key]] - out_edges = str(out_edges).replace("'", '"') + # out_edges = str(out_edges).replace("'", '"') back_edges = [f"{i}" for i in self.back_edges[key]] jump_target_str = f""" "{str(key)}": - type: {type(value)} + type: "{get_block_class_str(value)}" out: {out_edges}""" if back_edges: - back_edges = str(back_edges).replace("'", '"') + # back_edges = str(back_edges).replace("'", '"') jump_target_str += f""" back: {back_edges}""" yaml_string += dedent(jump_target_str) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 1f49f26..009970d 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -609,7 +609,7 @@ def join_returns(scfg: SCFG): # close if more than one is found if len(return_nodes) > 1: return_solo_label = SyntheticReturn() - scfg.insert_block(set(return_nodes), set(), block_label=return_solo_label) + scfg.insert_block(return_nodes, [], block_label=return_solo_label) def join_tails_and_exits(scfg: SCFG, tails: Set[BlockName], exits: Set[BlockName]): diff --git a/numba_rvsdg/tests/test_byteflow.py b/numba_rvsdg/tests/test_byteflow.py index 9f0030f..65be8d5 100644 --- a/numba_rvsdg/tests/test_byteflow.py +++ b/numba_rvsdg/tests/test_byteflow.py @@ -11,7 +11,7 @@ from numba_rvsdg.core.datastructures.byte_flow import ByteFlow from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.flow_info import FlowInfo - +from numba_rvsdg.tests.test_utils import SCFGComparator def fun(): x = 1 @@ -24,7 +24,7 @@ def fun(): class TestBCMapFromBytecode(unittest.TestCase): def test(self): # If the function definition line changes, just change the variable below, rest of it will adjust as long as function remains the same - func_def_line = 11 + func_def_line = 16 expected = { 0: Instruction( opname="RESUME", @@ -118,47 +118,23 @@ def test_constructor(self): label=PythonBytecodeLabel(), begin=0, end=8, - _jump_targets=(), - backedges=(), name_gen=name_gen, ) self.assertEqual(block.label, PythonBytecodeLabel()) self.assertEqual(block.begin, 0) self.assertEqual(block.end, 8) - self.assertFalse(block.fallthrough) - self.assertTrue(block.is_exiting) - self.assertEqual(block.jump_targets, ()) - self.assertEqual(block.backedges, ()) block_name = BlockName("pythonbytecodelabel_0") self.assertEqual(block.block_name, block_name) - def test_is_jump_target(self): - name_gen = NameGenerator(index=0) - label = PythonBytecodeLabel() - - block = PythonBytecodeBlock( - label=label, - begin=0, - end=8, - _jump_targets=(name_gen.new_block_name(label),), - backedges=(), - name_gen=name_gen, - ) - self.assertEqual(block.block_name, BlockName(name="pythonbytecodelabel_1")) - self.assertEqual(block.jump_targets, (BlockName(name="pythonbytecodelabel_0"),)) - self.assertFalse(block.is_exiting) - def test_get_instructions(self): # If the function definition line changes, just change the variable below, rest of it will adjust as long as function remains the same - func_def_line = 11 + func_def_line = 16 name_gen = NameGenerator(index=0) block = PythonBytecodeBlock( label=PythonBytecodeLabel(), begin=0, end=8, - _jump_targets=(), - backedges=(), name_gen=name_gen, ) expected = [ @@ -228,7 +204,7 @@ def test_get_instructions(self): ), ] - received = block.get_instructions(SCFG.bcmap_from_bytecode(bytecode)) + received = block.get_instructions(ByteFlow.bcmap_from_bytecode(bytecode)) self.assertEqual(expected, received) @@ -246,23 +222,15 @@ def test_from_bytecode(self): self.assertEqual(expected, received) def test_build_basic_blocks(self): - name_gen = NameGenerator(index=0) - expected = SCFG(graph={}) - block = PythonBytecodeBlock( - label=PythonBytecodeLabel(), - begin=0, - end=10, - _jump_targets=(), - backedges=(), - name_gen=name_gen, - ) - expected.add_block(block) + expected = SCFG() + expected.add_block("python_bytecode", get_label_class("python_bytecode")(), + begin=0, end=10) received = FlowInfo.from_bytecode(bytecode).build_basicblocks() self.assertEqual(expected, received) -class TestByteFlow(unittest.TestCase): +class TestByteFlow(SCFGComparator): def test_constructor(self): byteflow = ByteFlow([], []) self.assertEqual(len(byteflow.bc), 0) @@ -272,14 +240,14 @@ def test_from_bytecode(self): scfg = SCFG() scfg.add_block( - "python_bytecode", + block_type="python_bytecode", + block_label=get_label_class("python_bytecode")(), begin=0, end=10, - label=get_label_class("python_bytecode")(), ) expected = ByteFlow(bc=bytecode, scfg=scfg) received = ByteFlow.from_bytecode(fun) - self.assertEqual(expected.scfg, received.scfg) + self.assertSCFGEqual(expected.scfg, received.scfg) if __name__ == "__main__": diff --git a/numba_rvsdg/tests/test_scfg.py b/numba_rvsdg/tests/test_scfg.py index bc2abaf..e69ad07 100644 --- a/numba_rvsdg/tests/test_scfg.py +++ b/numba_rvsdg/tests/test_scfg.py @@ -2,12 +2,12 @@ from textwrap import dedent from numba_rvsdg.core.datastructures.scfg import SCFG -from numba_rvsdg.tests.test_utils import MapComparator +from numba_rvsdg.tests.test_utils import SCFGComparator from numba_rvsdg.core.datastructures.basic_block import BasicBlock from numba_rvsdg.core.datastructures.labels import Label, NameGenerator -class TestscfgConversion(MapComparator): +class TestSCFGConversion(SCFGComparator): def test_yaml_conversion(self): # Case # 1: Acyclic graph, no back-edges cases = [ @@ -70,8 +70,8 @@ def test_yaml_conversion(self): for case in cases: case = dedent(case) scfg, ref_dict = SCFG.from_yaml(case) - # TODO: use ref_dict for comparision - self.assertEqual(case, scfg.to_yaml()) + yaml = scfg.to_yaml() + self.assertYAMLEquals(case, yaml, ref_dict) def test_dict_conversion(self): # Case # 1: Acyclic graph, no back-edges @@ -104,10 +104,12 @@ def test_dict_conversion(self): for case in cases: scfg = SCFG.from_dict(case) - self.assertEqual(case, scfg.to_dict()) + scfg, ref_dict = SCFG.from_dict(case) + generated_dict = scfg.to_dict() + self.assertDictEquals(case, generated_dict, ref_dict) -class TestSCFGIterator(MapComparator): +class TestSCFGIterator(SCFGComparator): def test_scfg_iter(self): name_generator = NameGenerator() block_0 = BasicBlock(name_generator, Label()) @@ -116,7 +118,7 @@ def test_scfg_iter(self): (block_0.block_name, block_0), (block_1.block_name, block_1), ] - scfg = SCFG.from_yaml( + scfg, ref_dict = SCFG.from_yaml( """ "0": type: "basic" @@ -129,5 +131,200 @@ def test_scfg_iter(self): self.assertEqual(expected, received) +class TestInsertBlock(SCFGComparator): + def test_linear(self): + original = """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["2"] + "1": + type: "basic" + out: [] + "2": + type: "basic" + out: ["1"] + """ + expected_scfg, _ = SCFG.from_yaml(expected) + + preds = list((block_ref_orig["0"],)) + succs = list((block_ref_orig["1"],)) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) + + def test_dual_predecessor(self): + original = """ + "0": + type: "basic" + out: ["2"] + "1": + type: "basic" + out: ["2"] + "2": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["3"] + "1": + type: "basic" + out: ["3"] + "2": + type: "basic" + out: [] + "3": + type: "basic" + out: ["2"] + """ + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = list((block_ref_orig["0"], block_ref_orig["1"])) + succs = list((block_ref_orig["2"],)) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) + + def test_dual_successor(self): + original = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: [] + "2": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["3"] + "1": + type: "basic" + out: [] + "2": + type: "basic" + out: [] + "3": + type: "basic" + out: ["1", "2"] + """ + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = list((block_ref_orig["0"],)) + succs = list((block_ref_orig["1"], block_ref_orig["2"])) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) + + def test_dual_predecessor_and_dual_successor(self): + original = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["3"] + "2": + type: "basic" + out: ["4"] + "3": + type: "basic" + out: [] + "4": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["5"] + "2": + type: "basic" + out: ["5"] + "3": + type: "basic" + out: [] + "4": + type: "basic" + out: [] + "5": + type: "basic" + out: ["3", "4"] + """ + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = list((block_ref_orig["1"], block_ref_orig["2"])) + succs = list((block_ref_orig["3"], block_ref_orig["4"])) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) + + def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): + original = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["3"] + "2": + type: "basic" + out: ["1", "4"] + "3": + type: "basic" + out: ["0"] + "4": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["5"] + "2": + type: "basic" + out: ["1", "5"] + "3": + type: "basic" + out: ["0"] + "4": + type: "basic" + out: [] + "5": + type: "basic" + out: ["3", "4"] + """ + expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + + preds = list((block_ref_orig["1"], block_ref_orig["2"])) + succs = list((block_ref_orig["3"], block_ref_orig["4"])) + original_scfg.insert_block(preds, succs) + + self.assertSCFGEqual(expected_scfg, original_scfg) + + if __name__ == "__main__": main() diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index f579cdc..d7c0028 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -14,201 +14,6 @@ from numba_rvsdg.tests.test_utils import SCFGComparator -class TestInsertBlock(SCFGComparator): - def test_linear(self): - original = """ - "0": - type: "basic" - out: ["1"] - "1": - type: "basic" - out: [] - """ - original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected = """ - "0": - type: "basic" - out: ["2"] - "1": - type: "basic" - out: [] - "2": - type: "basic" - out: ["1"] - """ - expected_scfg, _ = SCFG.from_yaml(expected) - - preds = set((block_ref_orig["0"],)) - succs = set((block_ref_orig["1"],)) - original_scfg.insert_block(preds, succs) - - self.assertSCFGEqual(expected_scfg, original_scfg) - - def test_dual_predecessor(self): - original = """ - "0": - type: "basic" - out: ["2"] - "1": - type: "basic" - out: ["2"] - "2": - type: "basic" - out: [] - """ - original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected = """ - "0": - type: "basic" - out: ["3"] - "1": - type: "basic" - out: ["3"] - "2": - type: "basic" - out: [] - "3": - type: "basic" - out: ["2"] - """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - - preds = set((block_ref_orig["0"], block_ref_orig["1"])) - succs = set((block_ref_orig["2"],)) - original_scfg.insert_block(preds, succs) - - self.assertSCFGEqual(expected_scfg, original_scfg) - - def test_dual_successor(self): - original = """ - "0": - type: "basic" - out: ["1", "2"] - "1": - type: "basic" - out: [] - "2": - type: "basic" - out: [] - """ - original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected = """ - "0": - type: "basic" - out: ["3"] - "1": - type: "basic" - out: [] - "2": - type: "basic" - out: [] - "3": - type: "basic" - out: ["1", "2"] - """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - - preds = set((block_ref_orig["0"],)) - succs = set((block_ref_orig["1"], block_ref_orig["2"])) - original_scfg.insert_block(preds, succs) - - self.assertSCFGEqual(expected_scfg, original_scfg) - - def test_dual_predecessor_and_dual_successor(self): - original = """ - "0": - type: "basic" - out: ["1", "2"] - "1": - type: "basic" - out: ["3"] - "2": - type: "basic" - out: ["4"] - "3": - type: "basic" - out: [] - "4": - type: "basic" - out: [] - """ - original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected = """ - "0": - type: "basic" - out: ["1", "2"] - "1": - type: "basic" - out: ["5"] - "2": - type: "basic" - out: ["5"] - "3": - type: "basic" - out: [] - "4": - type: "basic" - out: [] - "5": - type: "basic" - out: ["3", "4"] - """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - - preds = set((block_ref_orig["1"], block_ref_orig["2"])) - succs = set((block_ref_orig["3"], block_ref_orig["4"])) - original_scfg.insert_block(preds, succs) - - self.assertSCFGEqual(expected_scfg, original_scfg) - - def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): - original = """ - "0": - type: "basic" - out: ["1", "2"] - "1": - type: "basic" - out: ["3"] - "2": - type: "basic" - out: ["1", "4"] - "3": - type: "basic" - out: ["0"] - "4": - type: "basic" - out: [] - """ - original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected = """ - "0": - type: "basic" - out: ["1", "2"] - "1": - type: "basic" - out: ["5"] - "2": - type: "basic" - out: ["1", "5"] - "3": - type: "basic" - out: ["0"] - "4": - type: "basic" - out: [] - "5": - type: "basic" - out: ["3", "4"] - """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - - preds = set((block_ref_orig["1"], block_ref_orig["2"])) - succs = set((block_ref_orig["3"], block_ref_orig["4"])) - original_scfg.insert_block(preds, succs) - - self.assertSCFGEqual(expected_scfg, original_scfg) - - class TestJoinReturns(SCFGComparator): def test_two_returns(self): original = """ @@ -266,8 +71,8 @@ def test_join_tails_and_exits_case_00(self): """ expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = set((block_ref_orig["0"],)) - exits = set((block_ref_orig["1"],)) + tails = list((block_ref_orig["0"],)) + exits = list((block_ref_orig["1"],)) join_tails_and_exits(original_scfg, tails, exits) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -308,8 +113,8 @@ def test_join_tails_and_exits_case_01(self): """ expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = set((block_ref_orig["0"],)) - exits = set((block_ref_orig["1"], block_ref_orig["2"])) + tails = list((block_ref_orig["0"],)) + exits = list((block_ref_orig["1"], block_ref_orig["2"])) join_tails_and_exits(original_scfg, tails, exits) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -350,8 +155,8 @@ def test_join_tails_and_exits_case_02_01(self): """ expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = set((block_ref_orig["1"], block_ref_orig["2"])) - exits = set((block_ref_orig["3"],)) + tails = list((block_ref_orig["1"], block_ref_orig["2"])) + exits = list((block_ref_orig["3"],)) join_tails_and_exits(original_scfg, tails, exits) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -392,8 +197,8 @@ def test_join_tails_and_exits_case_02_02(self): """ expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = set((block_ref_orig["1"], block_ref_orig["2"])) - exits = set((block_ref_orig["3"],)) + tails = list((block_ref_orig["1"], block_ref_orig["2"])) + exits = list((block_ref_orig["3"],)) join_tails_and_exits(original_scfg, tails, exits) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -451,8 +256,8 @@ def test_join_tails_and_exits_case_03_01(self): """ expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = set((block_ref_orig["1"], block_ref_orig["2"])) - exits = set((block_ref_orig["3"], block_ref_orig["4"])) + tails = list((block_ref_orig["1"], block_ref_orig["2"])) + exits = list((block_ref_orig["3"], block_ref_orig["4"])) join_tails_and_exits(original_scfg, tails, exits) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -510,8 +315,8 @@ def test_join_tails_and_exits_case_03_02(self): """ expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - tails = set((block_ref_orig["1"], block_ref_orig["2"])) - exits = set((block_ref_orig["3"], block_ref_orig["4"])) + tails = list((block_ref_orig["1"], block_ref_orig["2"])) + exits = list((block_ref_orig["3"], block_ref_orig["4"])) join_tails_and_exits(original_scfg, tails, exits) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -546,7 +351,7 @@ def test_no_op_mono(self): original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, set(wrap_id({"1"}))) + loop_restructure_helper(original_scfg, list(wrap_id({"1"}))) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -583,7 +388,7 @@ def test_no_op(self): """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, set(wrap_id({"1", "2"}))) + loop_restructure_helper(original_scfg, list(wrap_id({"1", "2"}))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_backedge_not_exiting(self): @@ -631,7 +436,7 @@ def test_backedge_not_exiting(self): """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, set(wrap_id({"1", "2"}))) + loop_restructure_helper(original_scfg, list(wrap_id({"1", "2"}))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_exit(self): @@ -689,7 +494,7 @@ def test_double_exit(self): """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, set(wrap_id({"1", "2", "3"}))) + loop_restructure_helper(original_scfg, list(wrap_id({"1", "2", "3"}))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_header(self): @@ -759,7 +564,7 @@ def test_double_header(self): """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, set(wrap_id({"1", "2", "3", "4"}))) + loop_restructure_helper(original_scfg, list(wrap_id({"1", "2", "3", "4"}))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_header_double_exiting(self): @@ -851,7 +656,7 @@ def test_double_header_double_exiting(self): """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, set(wrap_id({"1", "2", "3", "4"}))) + loop_restructure_helper(original_scfg, list(wrap_id({"1", "2", "3", "4"}))) self.assertSCFGEqual(expected_scfg, original_scfg) diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index 3346c02..f03c1f7 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -1,5 +1,6 @@ from unittest import TestCase +from typing import Dict, List from numba_rvsdg.core.datastructures.scfg import SCFG @@ -19,3 +20,17 @@ def assertSCFGEqual(self, first_scfg: SCFG, second_scfg: SCFG): self.assertEqual(first_scfg.out_edges[key1], second_scfg.out_edges[key2]) self.assertEqual(first_scfg.in_edges[key1], second_scfg.in_edges[key2]) self.assertEqual(first_scfg.back_edges[key1], second_scfg.back_edges[key2]) + + def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): + for key, value in ref_dict.items(): + second_yaml = second_yaml.replace(repr(value), key) + + self.assertEqual(first_yaml, second_yaml) + + def assertDictEquals(self, first_dict: str, second_dict: str, ref_dict: Dict): + for key, value in ref_dict.items(): + second_dict = second_dict.replace(repr(value), key) + + self.assertEqual(first_dict, second_dict) + + \ No newline at end of file From fcb948d5e8cddb91010dc76139b74ece8ac85c0b Mon Sep 17 00:00:00 2001 From: kc611 Date: Mon, 27 Mar 2023 19:28:13 +0530 Subject: [PATCH 05/30] Added precommit config --- .pre-commit-config.yaml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..23b60fc --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,40 @@ +exclude: | + (?x)^( + docs/.* + )$ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: debug-statements + - id: check-merge-conflict + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--py38-plus"] + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + language_version: python3 + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.11.5 + hooks: + - id: isort + - repo: https://github.com/humitos/mirrors-autoflake.git + rev: v1.1 + hooks: + - id: autoflake + args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable'] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.1.1 + hooks: + - id: mypy + additional_dependencies: + - types-filelock + - types-setuptools From 829ef611941799a3953ebace7bbca6ae9219006f Mon Sep 17 00:00:00 2001 From: kc611 Date: Tue, 28 Mar 2023 15:54:42 +0530 Subject: [PATCH 06/30] Refactored loop and branch restructuring --- numba_rvsdg/core/datastructures/scfg.py | 95 +++----- numba_rvsdg/core/transformations.py | 312 ++++++++++-------------- numba_rvsdg/rendering/rendering.py | 40 ++- numba_rvsdg/tests/test_scfg.py | 15 +- numba_rvsdg/tests/test_transforms.py | 46 +++- numba_rvsdg/tests/test_utils.py | 6 - 6 files changed, 226 insertions(+), 288 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index d107545..d1494ef 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -26,18 +26,14 @@ class SCFG: out_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict) back_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict) - in_edges: Dict[BlockName, List[BlockName]] = field( - default_factory=dict - ) # Design question - regions: Dict[RegionName, Region] = field(default_factory=dict) name_gen: NameGenerator = field(default_factory=NameGenerator, compare=False) - def __getitem__(self, index): + def __getitem__(self, index: BlockName) -> BasicBlock: return self.blocks[index] - def __contains__(self, index): + def __contains__(self, index: BlockName) -> bool: return index in self.blocks def __iter__(self): @@ -92,7 +88,7 @@ def __init__(self, graph): self.graph = graph def __getitem__(self, vertex): - out = out_edges[self.graph[vertex]] + out = out_edges[vertex] # Exclude node outside of the subgraph return [k for k in out if k in self.graph] @@ -115,7 +111,7 @@ def __init__(self, graph: Dict[BlockName, BasicBlock], subgraph): self.subgraph = subgraph def __getitem__(self, vertex): - out = out_edges[self.graph[vertex]] + out = out_edges[vertex] # Exclude node outside of the subgraph return [k for k in out if k in subgraph] @@ -142,7 +138,7 @@ def find_headers_and_entries( for outside in self.exclude_blocks(subgraph): nodes_jump_in_loop = subgraph.intersection( - self.out_edges[self.blocks[outside]] + self.out_edges[outside] ) headers.update(nodes_jump_in_loop) if nodes_jump_in_loop: @@ -169,19 +165,19 @@ def find_exiting_and_exits( exits: Set[BlockName] = set() for inside in subgraph: # any node inside that points outside the loop - for jt in self.out_edges[self.blocks[inside]]: + for jt in self.out_edges[inside]: if jt not in subgraph: exiting.add(inside) exits.add(jt) # any returns - if self.is_exiting(self.blocks[inside]): + if self.is_exiting(inside): exiting.add(inside) return exiting, exits def is_reachable_dfs(self, begin: BlockName, end: BlockName): # -> TypeGuard: """Is end reachable from begin.""" seen = set() - to_vist = list(self.out_edges[self.blocks[begin]]) + to_vist = list(self.out_edges[begin]) while True: if to_vist: block = to_vist.pop() @@ -195,7 +191,7 @@ def is_reachable_dfs(self, begin: BlockName, end: BlockName): # -> TypeGuard: elif block not in seen: seen.add(block) if block in self.blocks: - to_vist.extend(self.out_edges[self.blocks[block]]) + to_vist.extend(self.out_edges[block]) def is_exiting(self, block_name: BlockName): return len(self.out_edges[block_name]) == 0 @@ -212,73 +208,60 @@ def check_graph(self): # del self.blocks[name] # del self.out_edges[name] # del self.back_edges[name] - # del self.in_edges[name] # self.check_graph() - def insert_block( + def insert_block_between( self, + block_name: BlockName, predecessors: List[BlockName], - successors: List[BlockName], - block_type: str = "basic", - block_label: Label = Label(), - **block_args, + successors: List[BlockName] ): - # TODO: needs a diagram and documentaion - # initialize new block - - new_block_name = self.add_block( - block_type, block_label=block_label, **block_args - ) # Replace any arcs from any of predecessors to any of successors with # an arc through the inserted block instead. for pred_name in predecessors: # For every predecessor - # Remove all successors from its out edges - for successor in successors: - if successor in self.out_edges[pred_name]: - self.out_edges[pred_name].remove(successor) # Add the inserted block as out edge - self.out_edges[pred_name].append(new_block_name) - # For inserted block, the predecessor in an in-edge - self.in_edges[new_block_name].append(pred_name) + for idx, _out in enumerate(self.out_edges[pred_name]): + if _out in successors: + self.out_edges[pred_name][idx] = block_name + + if block_name not in self.out_edges[pred_name]: + self.out_edges[pred_name].append(block_name) + + self.out_edges[pred_name] = list(dict.fromkeys(self.out_edges[pred_name])) for success_name in successors: # For every sucessor - # Remove all predecessors from it's in edges - for pred in predecessors: - if pred in self.in_edges[success_name]: - self.in_edges[success_name].remove(pred) - # Add the inserted block as in edge - self.in_edges[success_name].append(new_block_name) # For inserted block, the sucessor in an out-edge - self.out_edges[new_block_name].append(success_name) + self.out_edges[block_name].append(success_name) self.check_graph() - return new_block_name def add_block( self, block_type: str = "basic", block_label: Label = Label(), **block_args ) -> BlockName: block_type = get_block_class(block_type) - new_block: BlockName = block_type(**block_args, label=block_label, name_gen=self.name_gen) + new_block: BasicBlock = block_type(**block_args, label=block_label, name_gen=self.name_gen) name = new_block.block_name self.blocks[name] = new_block - self.in_edges[name] = [] self.back_edges[name] = [] self.out_edges[name] = [] return name def add_connections(self, block_name, out_edges=[], back_edges=[]): + assert self.out_edges[block_name] == [] + assert self.back_edges[block_name] == [] self.out_edges[block_name] = out_edges self.back_edges[block_name] = back_edges - for edge in itertools.chain(out_edges, back_edges): - self.in_edges[edge].append(block_name) - self.check_graph() + + def add_region(self, region_head, region_exit, kind): + new_region = Region(self.name_gen, kind, region_head, region_exit) + self.regions[new_region.region_name] = new_region @staticmethod def from_yaml(yaml_string): @@ -332,13 +315,15 @@ def to_yaml(self): return yaml_string - # def to_dict(self): - # scfg_graph = self.graph - # graph_dict = {} - # for key, value in scfg_graph.items(): - # curr_dict = {} - # curr_dict["jt"] = [f"{i.index}" for i in value._out_edges] - # if value.backedges: - # curr_dict["be"] = [f"{i.index}" for i in value.backedges] - # graph_dict[str(key.index)] = curr_dict - # return graph_dict + def to_dict(self): + graph_dict = {} + for key, value in self.blocks.items(): + curr_dict = {} + curr_dict["type"] = get_block_class_str(value) + curr_dict["out"] = [f"{i}" for i in self.out_edges[key]] + back_edges = [f"{i}" for i in self.back_edges[key]] + if back_edges: + curr_dict["back"] = back_edges + graph_dict[str(key)] = curr_dict + + return graph_dict diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 009970d..c05c9cd 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -34,7 +34,7 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): ---------- scfg: SCFG The SCFG containing the loop - loop: Set[BlockName] + loop: List[BlockName] The loop (strongly connected components) that is to be restructured """ @@ -48,48 +48,49 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): # such that only a single loop header remains. if len(headers) > 1: headers_were_unified = True - solo_head_label = SyntheticHead(str(scfg.clg.new_index())) - insert_block_and_control_blocks(solo_head_label, entries, headers) - loop.add(solo_head_label) - loop_head: Label = solo_head_label + solo_head_label = SyntheticHead() + loop_head: BlockName = insert_block_and_control_blocks(scfg, list(entries), list(headers), block_label=solo_head_label) + loop.add(loop_head) else: - loop_head: Label = next(iter(headers)) + loop_head: BlockName = next(iter(headers)) # If there is only a single exiting latch (an exiting block that also has a # backedge to the loop header) we can exit early, since the condition for # SCFG is fullfilled. backedge_blocks = [ - block for block in loop if headers.intersection(scfg[block].jump_targets) + block for block in loop if set(headers).intersection(scfg.out_edges[block]) ] if ( len(backedge_blocks) == 1 and len(exiting_blocks) == 1 and backedge_blocks[0] == next(iter(exiting_blocks)) ): - scfg.add_block(scfg.graph.pop(backedge_blocks[0]).replace_backedge(loop_head)) + scfg.back_edges[backedge_blocks[0]].append(loop_head) return + doms = _doms(scfg) # The synthetic exiting latch and synthetic exit need to be created # based on the state of the cfg. If there are multiple exits, we need a # SyntheticExit, otherwise we only need a SyntheticExitingLatch - synth_exiting_latch = SyntheticExitingLatch(str(scfg.clg.new_index())) + # Set a flag, this will determine the variable assignment and block # insertion later on needs_synth_exit = len(exit_blocks) > 1 - if needs_synth_exit: - synth_exit = SyntheticExit(str(scfg.clg.new_index())) # This sets up the various control variables. # If there were multiple headers, we must re-use the variable that was used # for looping as the exit variable if headers_were_unified: - exit_variable = scfg[solo_head_label].variable + exit_variable = scfg[loop_head].variable else: - exit_variable = scfg.clg.new_variable() - # This variable denotes the backedge - backedge_variable = scfg.clg.new_variable() + exit_variable = scfg.name_gen.new_var_name() + + exit_value_table = dict(((i, j) for i, j in enumerate(exit_blocks))) + if needs_synth_exit: + synth_exit_label = SyntheticExit() + synth_exit = scfg.add_block("branch", block_label=synth_exit_label, variable=exit_variable, branch_value_table=exit_value_table) + # Now we setup the lookup tables for the various control variables, # depending on the state of the CFG and what is needed - exit_value_table = dict(((i, j) for i, j in enumerate(exit_blocks))) if needs_synth_exit: backedge_value_table = dict( (i, j) for i, j in enumerate((loop_head, synth_exit)) @@ -99,10 +100,15 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): (i, j) for i, j in enumerate((loop_head, next(iter(exit_blocks)))) ) if headers_were_unified: - header_value_table = scfg[solo_head_label].branch_value_table + header_value_table = scfg[loop_head].branch_value_table else: header_value_table = {} + synth_latch_label = SyntheticExitingLatch() + # This variable denotes the backedge + backedge_variable = scfg.name_gen.new_var_name() + synth_exiting_latch = scfg.add_block("branch", block_label=synth_latch_label, variable=backedge_variable, branch_value_table=backedge_value_table) + # This does a dictionary reverse lookup, to determine the key for a given # value. def reverse_lookup(d, value): @@ -114,22 +120,18 @@ def reverse_lookup(d, value): # Now that everything is in place, we can start to insert blocks, depending # on what is needed - # All new blocks are recorded for later insertion into the loop set - new_blocks = set() - doms = _doms(scfg) + # For every block in the loop: - for label in sorted(loop, key=lambda x: x.index): + for _name in loop.copy(): # If the block is an exiting block or a backedge block - if label in exiting_blocks or label in backedge_blocks: - # Copy the jump targets, these will be modified - new_jt = list(scfg[label].jump_targets) - # For each jump_target in the blockj - for jt in scfg[label].jump_targets: + if _name in exiting_blocks or _name in backedge_blocks: + # For each jump_target in the block + for jt in scfg.out_edges[_name]: # If the target is an exit block if jt in exit_blocks: - # Create a new assignment label and record it - synth_assign = SynthenticAssignment(str(scfg.clg.new_index())) - new_blocks.add(synth_assign) + # Create a new assignment name and record it + synth_assign = SynthenticAssignment() + # Setup the table for the variable assignment variable_assignment = {} # Setup the variables in the assignment table to point to @@ -142,24 +144,16 @@ def reverse_lookup(d, value): backedge_value_table, synth_exit if needs_synth_exit else next(iter(exit_blocks)), ) - # Create the actual control variable block - synth_assign_block = ControlVariableBlock( - label=synth_assign, - _jump_targets=(synth_exiting_latch,), - backedges=(), - variable_assignment=variable_assignment, - ) # Insert the assignment to the block map - scfg.add_block(synth_assign_block) - # Insert the new block into the new jump_targets making - # sure, that it replaces the correct jump_target, order - # matters in this case. - new_jt[new_jt.index(jt)] = synth_assign + synth_assign = scfg.add_block("control_variable", synth_assign, variable_assignment=variable_assignment) + scfg.add_connections(synth_assign, [synth_exiting_latch], []) + loop.add(synth_assign) + + scfg.out_edges[_name][scfg.out_edges[_name].index(jt)] = synth_assign # If the target is the loop_head - elif jt in headers and label not in doms[jt]: + elif jt in headers and _name not in doms[jt]: # Create the assignment and record it - synth_assign = SynthenticAssignment(str(scfg.clg.new_index())) - new_blocks.add(synth_assign) + synth_assign = SynthenticAssignment() # Setup the variables in the assignment table to point to # the correct blocks variable_assignment = {} @@ -170,59 +164,20 @@ def reverse_lookup(d, value): variable_assignment[exit_variable] = reverse_lookup( header_value_table, jt ) - # Update the backedge block - remove any existing backedges - # that point to the headers, no need to add a backedge, - # since it will be contained in the SyntheticExitingLatch - # later on. - block = scfg.graph.pop(label) - jts = list(block.jump_targets) - for h in headers: - if h in jts: - jts.remove(h) - scfg.add_block(block.replace_jump_targets(jump_targets=tuple(jts))) - # Setup the assignment block and initialize it with the - # correct jump_targets and variable assignment. - synth_assign_block = ControlVariableBlock( - label=synth_assign, - _jump_targets=(synth_exiting_latch,), - backedges=(), - variable_assignment=variable_assignment, - ) - # Add the new block to the SCFG - scfg.add_block(synth_assign_block) - # Update the jump targets again, order matters - new_jt[new_jt.index(jt)] = synth_assign - # finally, replace the jump_targets for this block with the new ones - scfg.add_block( - scfg.graph.pop(label).replace_jump_targets(jump_targets=tuple(new_jt)) - ) - # Add any new blocks to the loop. - loop.update(new_blocks) - - # Insert the exiting latch, add it to the loop and to the graph. - synth_exiting_latch_block = BranchBlock( - label=synth_exiting_latch, - _jump_targets=( - synth_exit if needs_synth_exit else next(iter(exit_blocks)), - loop_head, - ), - backedges=(loop_head,), - variable=backedge_variable, - branch_value_table=backedge_value_table, - ) + synth_assign = scfg.add_block("control_variable", synth_assign, variable_assignment=variable_assignment) + scfg.add_connections(synth_assign, [synth_exiting_latch], []) + loop.add(synth_assign) + + scfg.out_edges[_name][scfg.out_edges[_name].index(jt)] = synth_assign loop.add(synth_exiting_latch) - scfg.add_block(synth_exiting_latch_block) + # scfg.insert_block_between(synth_exiting_latch, list(exiting_blocks), list(exit_blocks)) # If an exit is to be created, we do so too, but only add it to the scfg, # since it isn't part of the loop if needs_synth_exit: - synth_exit_block = BranchBlock( - label=synth_exit, - _jump_targets=tuple(exit_blocks), - backedges=(), - variable=exit_variable, - branch_value_table=exit_value_table, - ) - scfg.add_block(synth_exit_block) + scfg.insert_block_between(synth_exit, [synth_exiting_latch], list(exit_blocks)) + + # Add the back_edge + scfg.back_edges[synth_exiting_latch].append(loop_head) def restructure_loop(scfg: SCFG): @@ -237,11 +192,11 @@ def restructure_loop(scfg: SCFG): loops: List[Set[SCFG]] = [ nodes for nodes in scc - if len(nodes) > 1 or next(iter(nodes)) in scfg[next(iter(nodes))].jump_targets + if len(nodes) > 1 or next(iter(nodes)) in scfg.out_edges[next(iter(nodes))] ] _logger.debug( - "restructure_loop found %d loops in %s", len(loops), scfg.graph.keys() + "restructure_loop found %d loops in %s", len(loops), scfg.blocks.keys() ) # rotate and extract loop for loop in loops: @@ -249,7 +204,7 @@ def restructure_loop(scfg: SCFG): extract_region(scfg, loop, "loop") -def find_head_blocks(scfg: SCFG, begin: Label) -> Set[Label]: +def find_head_blocks(scfg: SCFG, begin: BlockName) -> Set[BlockName]: head = scfg.find_head() head_region_blocks = set() current_block = head @@ -260,20 +215,20 @@ def find_head_blocks(scfg: SCFG, begin: Label) -> Set[Label]: if current_block == begin: break else: - jt = scfg.graph[current_block].jump_targets + jt = scfg.out_edges[current_block] assert len(jt) == 1 current_block = next(iter(jt)) return head_region_blocks -def find_branch_regions(scfg: SCFG, begin: Label, end: Label) -> Set[Label]: +def find_branch_regions(scfg: SCFG, begin: BlockName, end: BlockName) -> Set[BlockName]: # identify branch regions doms = _doms(scfg) postdoms = _post_doms(scfg) postimmdoms = _imm_doms(postdoms) immdoms = _imm_doms(doms) branch_regions = [] - jump_targets = scfg.graph[begin].jump_targets + jump_targets = scfg.out_edges[begin] for bra_start in jump_targets: for jt in jump_targets: if jt != bra_start and scfg.is_reachable_dfs(jt, bra_start): @@ -291,17 +246,17 @@ def find_branch_regions(scfg: SCFG, begin: Label, end: Label) -> Set[Label]: return branch_regions -def _find_branch_regions(scfg: SCFG, begin: Label, end: Label) -> Set[Label]: +def _find_branch_regions(scfg: SCFG, begin: BlockName, end: BlockName) -> Set[BlockName]: # identify branch regions branch_regions = [] - for bra_start in scfg[begin].jump_targets: + for bra_start in scfg.out_edges[begin]: region = [] region.append(bra_start) return branch_regions -def find_tail_blocks(scfg: SCFG, begin: Set[Label], head_region_blocks, branch_regions): - tail_subregion = set((b for b in scfg.graph.keys())) +def find_tail_blocks(scfg: SCFG, begin: Set[BlockName], head_region_blocks, branch_regions): + tail_subregion = set((b for b in scfg.blocks.keys())) tail_subregion.difference_update(head_region_blocks) for reg in branch_regions: if not reg: @@ -315,38 +270,19 @@ def find_tail_blocks(scfg: SCFG, begin: Set[Label], head_region_blocks, branch_r return tail_subregion -def extract_region(scfg, region_blocks, region_kind): +def extract_region(scfg: SCFG, region_blocks, region_kind): headers, entries = scfg.find_headers_and_entries(region_blocks) exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(region_blocks) assert len(headers) == 1 assert len(exiting_blocks) == 1 region_header = next(iter(headers)) region_exiting = next(iter(exiting_blocks)) - - head_subgraph = SCFG( - {label: scfg.graph[label] for label in region_blocks}, name_gen=scfg.name_gen - ) - - if isinstance(scfg[region_exiting], RegionBlock): - region_exit = scfg[region_exiting].exit - else: - region_exit = region_exiting - - subregion = RegionBlock( - label=region_header, - _jump_targets=scfg[region_exiting].jump_targets, - backedges=(), - kind=region_kind, - headers=headers, - subregion=head_subgraph, - exit=region_exit, - ) - scfg.remove_blocks(region_blocks) - scfg.graph[region_header] = subregion + + scfg.add_region(region_header, region_exiting, region_kind) def restructure_branch(scfg: SCFG): - print("restructure_branch", scfg.graph) + print("restructure_branch", scfg.blocks) doms = _doms(scfg) postdoms = _post_doms(scfg) postimmdoms = _imm_doms(postdoms) @@ -369,8 +305,8 @@ def restructure_branch(scfg: SCFG): # Unify headers of tail subregion if need be. headers, entries = scfg.find_headers_and_entries(tail_region_blocks) if len(headers) > 1: - end = SyntheticHead(scfg.name_gen.new_index()) - scfg.insert_block_and_control_blocks(end, entries, headers) + end = SyntheticHead() + insert_block_and_control_blocks(scfg, entries, headers, end) # Recompute regions. head_region_blocks = find_head_blocks(scfg, begin) @@ -389,15 +325,14 @@ def restructure_branch(scfg: SCFG): # Insert SyntheticTail exiting_blocks, _ = scfg.find_exiting_and_exits(inner_nodes) tail_headers, _ = scfg.find_headers_and_entries(tail_region_blocks) - _, _ = scfg.join_tails_and_exits(exiting_blocks, tail_headers) + _, _ = join_tails_and_exits(scfg, exiting_blocks, tail_headers) else: # Insert SyntheticBranch tail_headers, _ = scfg.find_headers_and_entries(tail_region_blocks) - synthetic_branch_block_label = SyntheticBranch( - str(scfg.name_gen.new_index()) - ) - scfg.insert_block(synthetic_branch_block_label, (begin,), tail_headers) + synthetic_branch_block_label = SyntheticBranch() + scfg.add_block(block_label=synthetic_branch_block_label) + scfg.insert_block_between(synthetic_branch_block_label, (begin,), tail_headers) # Recompute regions. head_region_blocks = find_head_blocks(scfg, begin) @@ -417,10 +352,10 @@ def restructure_branch(scfg: SCFG): def _iter_branch_regions( - scfg: SCFG, immdoms: Dict[Label, Label], postimmdoms: Dict[Label, Label] + scfg: SCFG, immdoms: Dict[BlockName, BlockName], postimmdoms: Dict[BlockName, BlockName] ): - for begin, node in [i for i in scfg.graph.items()]: - if len(node.jump_targets) > 1: + for begin, node in [i for i in scfg.blocks.items()]: + if len(scfg.out_edges[begin]) > 1: # found branch if begin in postimmdoms: end = postimmdoms[begin] @@ -428,7 +363,7 @@ def _iter_branch_regions( yield begin, end -def _imm_doms(doms: Dict[Label, Set[Label]]) -> Dict[Label, Label]: +def _imm_doms(doms: Dict[BlockName, Set[BlockName]]) -> Dict[BlockName, BlockName]: idoms = {k: v - {k} for k, v in doms.items()} changed = True while changed: @@ -455,36 +390,35 @@ def _doms(scfg: SCFG): succs_table = defaultdict(set) node: BasicBlock - for src, node in scfg.graph.items(): - for dst in node.jump_targets: + for src, node in scfg.blocks.items(): + for dst in scfg.out_edges[src]: # check dst is in subgraph - if dst in scfg.graph: + if dst in scfg.blocks: preds_table[dst].add(src) succs_table[src].add(dst) - for k in scfg.graph: + for k in scfg.blocks: if not preds_table[k]: entries.add(k) return _find_dominators_internal( - entries, list(scfg.graph.keys()), preds_table, succs_table + entries, list(scfg.blocks.keys()), preds_table, succs_table ) def _post_doms(scfg: SCFG): # compute post dom entries = set() - for k, v in scfg.blocks.items(): - targets = set(v.jump_targets) & set(scfg.graph) + for k in scfg.blocks.keys(): + targets = set(scfg.out_edges[k]) & set(scfg.blocks) if not targets: entries.add(k) preds_table = defaultdict(set) succs_table = defaultdict(set) - node: BasicBlock - for src, node in scfg.blocks.items(): - for dst in node.jump_targets: + for src in scfg.blocks.keys(): + for dst in scfg.out_edges[src]: # check dst is in subgraph - if dst in scfg.graph: + if dst in scfg.blocks: preds_table[src].add(dst) succs_table[dst].add(src) @@ -541,12 +475,10 @@ def _find_dominators_internal(entries, nodes, preds_table, succs_table): def insert_block_and_control_blocks( scfg: SCFG, - predecessors: Set[BlockName], - successors: Set[BlockName], - block_type: str = "basic", - block_label: Label = Label(), - **block_args -): + predecessors: List[BlockName], + successors: List[BlockName], + block_label: Label = Label() +) -> BlockName: # TODO: needs a diagram and documentaion # name of the variable for this branching assignment branch_variable = scfg.name_gen.new_var_name() @@ -558,11 +490,11 @@ def insert_block_and_control_blocks( branch_block_name = scfg.add_block( "branch", block_label, - branch_variable=branch_variable, - branch_value_table=branch_value_table, + variable=branch_variable, + branch_value_table=branch_value_table ) - scfg.add_connections(branch_block_name, tuple(successors), set()) + control_blocks = {} # Replace any arcs from any of predecessors to any of successors with # an arc through the to be inserted block instead. @@ -571,30 +503,29 @@ def insert_block_and_control_blocks( # Need to create synthetic assignments for each arc from a # predecessors to a successor and insert it between the predecessor # and the newly created block - for s in pred_outs.intersection(successors): - synth_assign = SynthenticAssignment() - variable_assignment = {} - variable_assignment[branch_variable] = branch_variable_value - - # add block - control_block_name = scfg.add_block( - "control_variable", - synth_assign, - variable_assignment=variable_assignment, - ) - scfg.add_connections(control_block_name, set((branch_block_name,)), set()) - # update branching table - branch_value_table[branch_variable_value] = s - # update branching variable - branch_variable_value += 1 - - # Rewrite old edges - scfg.out_edges[pred_name].remove(s) - scfg.out_edges[pred_name].add(control_block_name) - - scfg.in_edges[s].remove(pred_name) - scfg.in_edges[s].add(control_block_name) - + for s in successors: + if s in pred_outs: + synth_assign = SynthenticAssignment() + variable_assignment = {} + variable_assignment[branch_variable] = branch_variable_value + + # add block + control_block_name = scfg.add_block( + "control_variable", + synth_assign, + variable_assignment=variable_assignment, + ) + # update branching table + branch_value_table[branch_variable_value] = s + # update branching variable + branch_variable_value += 1 + control_blocks[control_block_name] = pred_name + + scfg.insert_block_between(branch_block_name, predecessors, successors) + + for _synth_assign, _pred in control_blocks.items(): + scfg.insert_block_between(_synth_assign, [_pred], [branch_block_name]) + return branch_block_name @@ -609,7 +540,8 @@ def join_returns(scfg: SCFG): # close if more than one is found if len(return_nodes) > 1: return_solo_label = SyntheticReturn() - scfg.insert_block(return_nodes, [], block_label=return_solo_label) + new_block = scfg.add_block(block_label=return_solo_label) + scfg.insert_block_between(new_block, return_nodes, []) def join_tails_and_exits(scfg: SCFG, tails: Set[BlockName], exits: Set[BlockName]): @@ -623,22 +555,26 @@ def join_tails_and_exits(scfg: SCFG, tails: Set[BlockName], exits: Set[BlockName # join only exits solo_tail_name = next(iter(tails)) solo_exit_label = SyntheticExit() - solo_exit_name = scfg.insert_block(tails, exits, block_label=solo_exit_label) + solo_exit_name = scfg.add_block(block_label=solo_exit_label) + scfg.insert_block_between(solo_exit_name, tails, exits) return solo_tail_name, solo_exit_name if len(tails) >= 2 and len(exits) == 1: # join only tails solo_tail_label = SyntheticTail() solo_exit_name = next(iter(exits)) - solo_tail_name = scfg.insert_block(tails, exits, block_label=solo_tail_label) + solo_tail_name = scfg.add_block(block_label=solo_tail_label) + scfg.insert_block_between(solo_tail_name, tails, exits) return solo_tail_name, solo_exit_name if len(tails) >= 2 and len(exits) >= 2: # join both tails and exits solo_tail_label = SyntheticTail() solo_exit_label = SyntheticExit() - solo_tail_name = scfg.insert_block(tails, exits, block_label=solo_tail_label) - solo_exit_name = scfg.insert_block( - set((solo_tail_name,)), exits, block_label=solo_exit_label - ) + + solo_tail_name = scfg.add_block(block_label=solo_tail_label) + scfg.insert_block_between(solo_tail_name, tails, exits) + + solo_exit_name = scfg.add_block(block_label=solo_exit_label) + scfg.insert_block_between(solo_exit_name, set((solo_tail_name,)), exits) return solo_tail_name, solo_exit_name diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 3619974..3cbfb4c 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -32,21 +32,17 @@ def __init__(self, byte_flow: ByteFlow): def render_regions(self): # render subgraph - # graph = regionblock.get_full_graph() - # with digraph.subgraph(name=f"cluster_{label}") as subg: - # color = "blue" - # if regionblock.kind == "branch": - # color = "green" - # if regionblock.kind == "tail": - # color = "purple" - # if regionblock.kind == "head": - # color = "red" - # subg.attr(color=color, label=regionblock.kind) - # for label, block in graph.items(): - # self.render_block(subg, label, block) - # # render edges within this region - # self.render_edges(graph) - pass + for region_name, region in self.scfg.regions.items(): + graph = region.get_full_graph() + with self.g.subgraph(name=f"cluster_{label}") as subg: + color = "blue" + if region.kind == "branch": + color = "green" + if region.kind == "tail": + color = "purple" + if region.kind == "head": + color = "red" + subg.attr(color=color, label=region.kind) def render_basic_block(self, block_name: BlockName): block = self.scfg[block_name] @@ -68,9 +64,9 @@ def render_control_variable_block(self, block_name: BlockName): if isinstance(block.label, ControlLabel): body = str(block_name) + "\l" - body += "\l".join( - (f"{k} = {v}" for k, v in block.variable_assignment.items()) - ) + # body += "\l".join( + # (f"{k} = {v}" for k, v in block.variable_assignment.items()) + # ) else: raise Exception("Unknown label type: " + block.label) self.g.node(str(block_name), shape="rect", label=body) @@ -87,10 +83,10 @@ def find_index(v): return v.index body = str(block_name) + "\l" - body += f"variable: {block.variable}\l" - body += "\l".join( - (f"{k}=>{find_index(v)}" for k, v in block.branch_value_table.items()) - ) + # body += f"variable: {block.variable}\l" + # body += "\l".join( + # (f" {k} => {find_index(v)}" for k, v in block.branch_value_table.items()) + # ) else: raise Exception("Unknown label type: " + block.label) self.g.node(str(block_name), shape="rect", label=body) diff --git a/numba_rvsdg/tests/test_scfg.py b/numba_rvsdg/tests/test_scfg.py index e69ad07..1717121 100644 --- a/numba_rvsdg/tests/test_scfg.py +++ b/numba_rvsdg/tests/test_scfg.py @@ -157,7 +157,8 @@ def test_linear(self): preds = list((block_ref_orig["0"],)) succs = list((block_ref_orig["1"],)) - original_scfg.insert_block(preds, succs) + new_block = original_scfg.add_block() + original_scfg.insert_block_between(new_block, preds, succs) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -192,7 +193,8 @@ def test_dual_predecessor(self): preds = list((block_ref_orig["0"], block_ref_orig["1"])) succs = list((block_ref_orig["2"],)) - original_scfg.insert_block(preds, succs) + new_block = original_scfg.add_block() + original_scfg.insert_block_between(new_block, preds, succs) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -227,7 +229,8 @@ def test_dual_successor(self): preds = list((block_ref_orig["0"],)) succs = list((block_ref_orig["1"], block_ref_orig["2"])) - original_scfg.insert_block(preds, succs) + new_block = original_scfg.add_block() + original_scfg.insert_block_between(new_block, preds, succs) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -274,7 +277,8 @@ def test_dual_predecessor_and_dual_successor(self): preds = list((block_ref_orig["1"], block_ref_orig["2"])) succs = list((block_ref_orig["3"], block_ref_orig["4"])) - original_scfg.insert_block(preds, succs) + new_block = original_scfg.add_block() + original_scfg.insert_block_between(new_block, preds, succs) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -321,7 +325,8 @@ def test_dual_predecessor_and_dual_successor_with_additional_arcs(self): preds = list((block_ref_orig["1"], block_ref_orig["2"])) succs = list((block_ref_orig["3"], block_ref_orig["4"])) - original_scfg.insert_block(preds, succs) + new_block = original_scfg.add_block() + original_scfg.insert_block_between(new_block, preds, succs) self.assertSCFGEqual(expected_scfg, original_scfg) diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index d7c0028..acabb68 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -343,7 +343,7 @@ def test_no_op_mono(self): "1": type: "basic" out: ["1", "2"] - be: ["1"] + back: ["1"] "2": type: "basic" out: [] @@ -351,7 +351,7 @@ def test_no_op_mono(self): original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, list(wrap_id({"1"}))) + loop_restructure_helper(original_scfg, set((block_ref_orig["1"],))) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -381,14 +381,14 @@ def test_no_op(self): "2": type: "basic" out: ["1", "3"] - be: ["1"] + back: ["1"] "3": type: "basic" out: [] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, list(wrap_id({"1", "2"}))) + loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"]))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_backedge_not_exiting(self): @@ -426,17 +426,19 @@ def test_backedge_not_exiting(self): "4": type: "basic" out: ["1", "3"] - be: ["1"] + back: ["1"] "5": type: "basic" + label_type: "synth_assign" out: ["4"] "6": type: "basic" + label_type: "synth_assign" out: ["4"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, list(wrap_id({"1", "2"}))) + loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"]))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_exit(self): @@ -480,21 +482,25 @@ def test_double_exit(self): out: [] "5": type: "basic" + label_type: "synth_exit_latch" out: ["1", "4"] - be: ["1"] + back: ["1"] "6": type: "basic" + label_type: "synth_assign" out: ["5"] "7": type: "basic" + label_type: "synth_assign" out: ["5"] "8": type: "basic" + label_type: "synth_assign" out: ["5"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, list(wrap_id({"1", "2", "3"}))) + loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"]))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_header(self): @@ -541,30 +547,37 @@ def test_double_header(self): out: [] "6": type: "basic" + label_type: "synth_head" out: ["1", "2"] "7": type: "basic" + label_type: "synth_assign" out: ["6"] "8": type: "basic" + label_type: "synth_assign" out: ["6"] "9": type: "basic" + label_type: "synth_exit_latch" out: ["5", "6"] - be: ["6"] + back: ["6"] "10": type: "basic" + label_type: "synth_assign" out: ["9"] "11": type: "basic" + label_type: "synth_assign" out: ["9"] "12": type: "basic" + label_type: "synth_assign" out: ["9"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, list(wrap_id({"1", "2", "3", "4"}))) + loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"], block_ref_orig["4"]))) self.assertSCFGEqual(expected_scfg, original_scfg) def test_double_header_double_exiting(self): @@ -627,36 +640,45 @@ def test_double_header_double_exiting(self): out: [] "8": type: "basic" + label_type: "synth_head" out: ["1", "2"] "9": type: "basic" + label_type: "synth_assign" out: ["8"] "10": type: "basic" + label_type: "synth_assign" out: ["8"] "11": type: "basic" + label_type: "synth_exit" out: ["12", "8"] - be: ["8"] + back: ["8"] "12": type: "basic" + label_type: "synth_exit_latch" out: ["5", "6"] "13": type: "basic" + label_type: "synth_assign" out: ["11"] "14": type: "basic" + label_type: "synth_assign" out: ["11"] "15": type: "basic" + label_type: "synth_assign" out: ["11"] "16": type: "basic" + label_type: "synth_assign" out: ["11"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) - loop_restructure_helper(original_scfg, list(wrap_id({"1", "2", "3", "4"}))) + loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"], block_ref_orig["4"]))) self.assertSCFGEqual(expected_scfg, original_scfg) diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index f03c1f7..995dafb 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -18,7 +18,6 @@ def assertSCFGEqual(self, first_scfg: SCFG, second_scfg: SCFG): self.assertEqual(type(block_1.label), type(block_2.label)) # compare edges self.assertEqual(first_scfg.out_edges[key1], second_scfg.out_edges[key2]) - self.assertEqual(first_scfg.in_edges[key1], second_scfg.in_edges[key2]) self.assertEqual(first_scfg.back_edges[key1], second_scfg.back_edges[key2]) def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): @@ -28,9 +27,4 @@ def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): self.assertEqual(first_yaml, second_yaml) def assertDictEquals(self, first_dict: str, second_dict: str, ref_dict: Dict): - for key, value in ref_dict.items(): - second_dict = second_dict.replace(repr(value), key) - self.assertEqual(first_dict, second_dict) - - \ No newline at end of file From 5a00f5d748faf8fc1f6250ff85fe66862f2878ce Mon Sep 17 00:00:00 2001 From: kc611 Date: Tue, 28 Mar 2023 16:14:23 +0530 Subject: [PATCH 07/30] Fixed dictionary conversion testing --- numba_rvsdg/tests/test_utils.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index 995dafb..6561791 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -26,5 +26,21 @@ def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): self.assertEqual(first_yaml, second_yaml) - def assertDictEquals(self, first_dict: str, second_dict: str, ref_dict: Dict): + def assertDictEquals(self, first_dict: dict, second_dict: dict, ref_dict: dict): + + def replace_with_refs(scfg_dict: dict): + new_dict = {} + for key, value in scfg_dict.items(): + key = str(ref_dict[key]) + _new_dict = {} + for _key, _value in value.items(): + if isinstance(_value, list): + for i in range(len(_value)): + _value[i] = str(ref_dict[_value[i]]) + _new_dict[_key] = _value + new_dict[key] = _new_dict + + return new_dict + + first_dict = replace_with_refs(first_dict) self.assertEqual(first_dict, second_dict) From 96b03918f5bfa74dcbad224348de8fb7765cb6a2 Mon Sep 17 00:00:00 2001 From: kc611 Date: Tue, 28 Mar 2023 20:04:23 +0530 Subject: [PATCH 08/30] Fixed failing tests in test_transforms --- numba_rvsdg/core/transformations.py | 15 ++++++++++----- numba_rvsdg/rendering/rendering.py | 23 ++++++++++++----------- numba_rvsdg/tests/test_transforms.py | 19 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index c05c9cd..330fb05 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -43,13 +43,14 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(loop) # assert len(entries) == 1 headers_were_unified = False + exit_blocks = list(sorted(exit_blocks)) # If there are multiple headers, insert assignment and control blocks, # such that only a single loop header remains. if len(headers) > 1: headers_were_unified = True solo_head_label = SyntheticHead() - loop_head: BlockName = insert_block_and_control_blocks(scfg, list(entries), list(headers), block_label=solo_head_label) + loop_head: BlockName = insert_block_and_control_blocks(scfg, list(sorted(entries)), list(sorted(headers)), block_label=solo_head_label) loop.add(loop_head) else: loop_head: BlockName = next(iter(headers)) @@ -122,7 +123,7 @@ def reverse_lookup(d, value): # on what is needed # For every block in the loop: - for _name in loop.copy(): + for _name in sorted(loop): # If the block is an exiting block or a backedge block if _name in exiting_blocks or _name in backedge_blocks: # For each jump_target in the block @@ -170,14 +171,18 @@ def reverse_lookup(d, value): scfg.out_edges[_name][scfg.out_edges[_name].index(jt)] = synth_assign loop.add(synth_exiting_latch) + + # Add the back_edge + scfg.out_edges[synth_exiting_latch].append(loop_head) + scfg.back_edges[synth_exiting_latch].append(loop_head) + # scfg.insert_block_between(synth_exiting_latch, list(exiting_blocks), list(exit_blocks)) # If an exit is to be created, we do so too, but only add it to the scfg, # since it isn't part of the loop if needs_synth_exit: scfg.insert_block_between(synth_exit, [synth_exiting_latch], list(exit_blocks)) - - # Add the back_edge - scfg.back_edges[synth_exiting_latch].append(loop_head) + else: + scfg.out_edges[synth_exiting_latch].append(list(exit_blocks)[0]) def restructure_loop(scfg: SCFG): diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 3cbfb4c..1f15f63 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -32,17 +32,18 @@ def __init__(self, byte_flow: ByteFlow): def render_regions(self): # render subgraph - for region_name, region in self.scfg.regions.items(): - graph = region.get_full_graph() - with self.g.subgraph(name=f"cluster_{label}") as subg: - color = "blue" - if region.kind == "branch": - color = "green" - if region.kind == "tail": - color = "purple" - if region.kind == "head": - color = "red" - subg.attr(color=color, label=region.kind) + # for region_name, region in self.scfg.regions.items(): + # graph = region.get_full_graph() + # with self.g.subgraph(name=f"cluster_{label}") as subg: + # color = "blue" + # if region.kind == "branch": + # color = "green" + # if region.kind == "tail": + # color = "purple" + # if region.kind == "head": + # color = "red" + # subg.attr(color=color, label=region.kind) + pass def render_basic_block(self, block_name: BlockName): block = self.scfg[block_name] diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index acabb68..67441e7 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -425,6 +425,7 @@ def test_backedge_not_exiting(self): out: [] "4": type: "basic" + label_type: "synth_exit_latch" out: ["1", "3"] back: ["1"] "5": @@ -560,7 +561,7 @@ def test_double_header(self): "9": type: "basic" label_type: "synth_exit_latch" - out: ["5", "6"] + out: ["6", "5"] back: ["6"] "10": type: "basic" @@ -616,7 +617,7 @@ def test_double_header_double_exiting(self): expected = """ "0": type: "basic" - out: ["10", "9"] + out: ["9", "10"] "1": type: "basic" out: ["3"] @@ -653,28 +654,28 @@ def test_double_header_double_exiting(self): "11": type: "basic" label_type: "synth_exit" - out: ["12", "8"] - back: ["8"] + out: ["5", "6"] "12": type: "basic" label_type: "synth_exit_latch" - out: ["5", "6"] + out: ["8", "11"] + back: ["8"] "13": type: "basic" label_type: "synth_assign" - out: ["11"] + out: ["12"] "14": type: "basic" label_type: "synth_assign" - out: ["11"] + out: ["12"] "15": type: "basic" label_type: "synth_assign" - out: ["11"] + out: ["12"] "16": type: "basic" label_type: "synth_assign" - out: ["11"] + out: ["12"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) expected_scfg, block_ref_exp = SCFG.from_yaml(expected) From bf4a9031f55fb1e7069e83652fd3818619bd74fb Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 13:52:52 +0200 Subject: [PATCH 09/30] fix trailing whitespace and debugging leftovers As title --- numba_rvsdg/core/datastructures/scfg.py | 4 ++-- numba_rvsdg/core/transformations.py | 8 +++----- numba_rvsdg/tests/test_utils.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index d1494ef..5110f0f 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -227,7 +227,7 @@ def insert_block_between( if block_name not in self.out_edges[pred_name]: self.out_edges[pred_name].append(block_name) - + self.out_edges[pred_name] = list(dict.fromkeys(self.out_edges[pred_name])) for success_name in successors: @@ -258,7 +258,7 @@ def add_connections(self, block_name, out_edges=[], back_edges=[]): self.back_edges[block_name] = back_edges self.check_graph() - + def add_region(self, region_head, region_exit, kind): new_region = Region(self.name_gen, kind, region_head, region_exit) self.regions[new_region.region_name] = new_region diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 330fb05..1eee80a 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -41,7 +41,6 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): headers, entries = scfg.find_headers_and_entries(loop) exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(loop) - # assert len(entries) == 1 headers_were_unified = False exit_blocks = list(sorted(exit_blocks)) @@ -176,7 +175,6 @@ def reverse_lookup(d, value): scfg.out_edges[synth_exiting_latch].append(loop_head) scfg.back_edges[synth_exiting_latch].append(loop_head) - # scfg.insert_block_between(synth_exiting_latch, list(exiting_blocks), list(exit_blocks)) # If an exit is to be created, we do so too, but only add it to the scfg, # since it isn't part of the loop if needs_synth_exit: @@ -282,7 +280,7 @@ def extract_region(scfg: SCFG, region_blocks, region_kind): assert len(exiting_blocks) == 1 region_header = next(iter(headers)) region_exiting = next(iter(exiting_blocks)) - + scfg.add_region(region_header, region_exiting, region_kind) @@ -525,12 +523,12 @@ def insert_block_and_control_blocks( # update branching variable branch_variable_value += 1 control_blocks[control_block_name] = pred_name - + scfg.insert_block_between(branch_block_name, predecessors, successors) for _synth_assign, _pred in control_blocks.items(): scfg.insert_block_between(_synth_assign, [_pred], [branch_block_name]) - + return branch_block_name diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index 6561791..97aee1f 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -25,7 +25,7 @@ def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): second_yaml = second_yaml.replace(repr(value), key) self.assertEqual(first_yaml, second_yaml) - + def assertDictEquals(self, first_dict: dict, second_dict: dict, ref_dict: dict): def replace_with_refs(scfg_dict: dict): From 779b0cbb20f64f30cd6bba75cac351fa7a517b60 Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 14:01:48 +0200 Subject: [PATCH 10/30] refactor find_head_blocks Rename the variable that is bound to the outgoing edges and use the list indexing to access the elements. --- numba_rvsdg/core/transformations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 1eee80a..0164d4d 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -218,9 +218,9 @@ def find_head_blocks(scfg: SCFG, begin: BlockName) -> Set[BlockName]: if current_block == begin: break else: - jt = scfg.out_edges[current_block] - assert len(jt) == 1 - current_block = next(iter(jt)) + out_targets = scfg.out_edges[current_block] + assert len(out_targets) == 1 + current_block = out_targets[0] return head_region_blocks From 951a8379597650434e222073d8222965a07660bc Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 14:05:47 +0200 Subject: [PATCH 11/30] refactor find_branch_regions As title --- numba_rvsdg/core/transformations.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 0164d4d..f721cb9 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -231,10 +231,11 @@ def find_branch_regions(scfg: SCFG, begin: BlockName, end: BlockName) -> Set[Blo postimmdoms = _imm_doms(postdoms) immdoms = _imm_doms(doms) branch_regions = [] - jump_targets = scfg.out_edges[begin] - for bra_start in jump_targets: - for jt in jump_targets: - if jt != bra_start and scfg.is_reachable_dfs(jt, bra_start): + out_targets = scfg.out_edges[begin] + for bra_start in out_targets: + for out_targets in jump_targets: + if (out_targets != bra_start + and scfg.is_reachable_dfs(out_targets, bra_start)): branch_regions.append(tuple()) break else: From 4647610e034d0fbfc196852ccb11e1c0af90dfcc Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 14:16:16 +0200 Subject: [PATCH 12/30] refactor loop_restructure_helper As title --- numba_rvsdg/core/transformations.py | 69 ++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index f721cb9..56f5bdd 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -49,7 +49,11 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): if len(headers) > 1: headers_were_unified = True solo_head_label = SyntheticHead() - loop_head: BlockName = insert_block_and_control_blocks(scfg, list(sorted(entries)), list(sorted(headers)), block_label=solo_head_label) + loop_head: BlockName = insert_block_and_control_blocks( + scfg, + list(sorted(entries)), + list(sorted(headers)), + block_label=solo_head_label) loop.add(loop_head) else: loop_head: BlockName = next(iter(headers)) @@ -57,7 +61,8 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): # backedge to the loop header) we can exit early, since the condition for # SCFG is fullfilled. backedge_blocks = [ - block for block in loop if set(headers).intersection(scfg.out_edges[block]) + block for block in loop + if set(headers).intersection(scfg.out_edges[block]) ] if ( len(backedge_blocks) == 1 @@ -87,7 +92,11 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): exit_value_table = dict(((i, j) for i, j in enumerate(exit_blocks))) if needs_synth_exit: synth_exit_label = SyntheticExit() - synth_exit = scfg.add_block("branch", block_label=synth_exit_label, variable=exit_variable, branch_value_table=exit_value_table) + synth_exit = scfg.add_block( + "branch", + block_label=synth_exit_label, + variable=exit_variable, + branch_value_table=exit_value_table) # Now we setup the lookup tables for the various control variables, # depending on the state of the CFG and what is needed @@ -107,7 +116,11 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): synth_latch_label = SyntheticExitingLatch() # This variable denotes the backedge backedge_variable = scfg.name_gen.new_var_name() - synth_exiting_latch = scfg.add_block("branch", block_label=synth_latch_label, variable=backedge_variable, branch_value_table=backedge_value_table) + synth_exiting_latch = scfg.add_block( + "branch", + block_label=synth_latch_label, + variable=backedge_variable, + branch_value_table=backedge_value_table) # This does a dictionary reverse lookup, to determine the key for a given # value. @@ -126,9 +139,9 @@ def reverse_lookup(d, value): # If the block is an exiting block or a backedge block if _name in exiting_blocks or _name in backedge_blocks: # For each jump_target in the block - for jt in scfg.out_edges[_name]: + for out_target in scfg.out_edges[_name]: # If the target is an exit block - if jt in exit_blocks: + if out_target in exit_blocks: # Create a new assignment name and record it synth_assign = SynthenticAssignment() @@ -138,20 +151,28 @@ def reverse_lookup(d, value): # the correct blocks if needs_synth_exit: variable_assignment[exit_variable] = reverse_lookup( - exit_value_table, jt + exit_value_table, out_target ) variable_assignment[backedge_variable] = reverse_lookup( backedge_value_table, synth_exit if needs_synth_exit else next(iter(exit_blocks)), ) # Insert the assignment to the block map - synth_assign = scfg.add_block("control_variable", synth_assign, variable_assignment=variable_assignment) - scfg.add_connections(synth_assign, [synth_exiting_latch], []) + synth_assign = scfg.add_block( + "control_variable", + synth_assign, + variable_assignment=variable_assignment) + scfg.add_connections( + synth_assign, + [synth_exiting_latch], + []) loop.add(synth_assign) - - scfg.out_edges[_name][scfg.out_edges[_name].index(jt)] = synth_assign + # Update the edge from the out_target to point to the new + # assignment block + out_edges = scfg.out_edges[_name] + out_edges[out_edges.index(out_target)] = synth_assign # If the target is the loop_head - elif jt in headers and _name not in doms[jt]: + elif out_target in headers and _name not in doms[out_target]: # Create the assignment and record it synth_assign = SynthenticAssignment() # Setup the variables in the assignment table to point to @@ -162,13 +183,24 @@ def reverse_lookup(d, value): ) if needs_synth_exit: variable_assignment[exit_variable] = reverse_lookup( - header_value_table, jt + header_value_table, out_target ) - synth_assign = scfg.add_block("control_variable", synth_assign, variable_assignment=variable_assignment) - scfg.add_connections(synth_assign, [synth_exiting_latch], []) + synth_assign = scfg.add_block( + "control_variable", + synth_assign, + variable_assignment=variable_assignment) + scfg.add_connections( + synth_assign, + [synth_exiting_latch], + []) loop.add(synth_assign) - scfg.out_edges[_name][scfg.out_edges[_name].index(jt)] = synth_assign + # Update the edge from the out_target to point to the new + # assignment block + out_edges = scfg.out_edges[_name] + out_edges[out_edges.index(out_target)] = synth_assign + + # Finally, add the synthetic exiting latch to loop loop.add(synth_exiting_latch) # Add the back_edge @@ -178,7 +210,10 @@ def reverse_lookup(d, value): # If an exit is to be created, we do so too, but only add it to the scfg, # since it isn't part of the loop if needs_synth_exit: - scfg.insert_block_between(synth_exit, [synth_exiting_latch], list(exit_blocks)) + scfg.insert_block_between( + synth_exit, + [synth_exiting_latch], + list(exit_blocks)) else: scfg.out_edges[synth_exiting_latch].append(list(exit_blocks)[0]) From fe664b9ca138cc676955f4638ad4e483d2701250 Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 14:39:13 +0200 Subject: [PATCH 13/30] add typing and refactor NameGenerator Adds the typing signatures to the NameGenerator, adds numpydoc compatible documentation and does some refactoring on the code, mostly variable names and shortening one construct. --- numba_rvsdg/core/datastructures/labels.py | 32 ++++++++++++++++------- numba_rvsdg/tests/test_byteflow.py | 4 +-- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/numba_rvsdg/core/datastructures/labels.py b/numba_rvsdg/core/datastructures/labels.py index a23debd..6aadb6c 100644 --- a/numba_rvsdg/core/datastructures/labels.py +++ b/numba_rvsdg/core/datastructures/labels.py @@ -96,21 +96,33 @@ class RegionName: @dataclass class NameGenerator: - index: int = 0 - var_index: int = 0 + """Name generator for various element names. + + Attributes + ---------- + + block_index : int + The starting index for blocks + variable_index: int + The starting index for control variables + region_index : int + The starting index for regions + """ + block_index: int = 0 + variable_index: int = 97 # Variables start at lowercase 'a' region_index: int = 0 - def new_block_name(self, label): - ret = self.index - self.index += 1 + def new_block_name(self, label: str) -> BlockName: + ret = self.block_index + self.block_index += 1 return BlockName(str(label).lower().split("(")[0] + "_" + str(ret)) - def new_region_name(self, kind): + def new_region_name(self, kind: str) -> RegionName: ret = self.region_index self.region_index += 1 return RegionName(str(kind).lower().split("(")[0] + "_" + str(ret)) - def new_var_name(self): - var_name = chr(self.var_index) - self.var_index = self.var_index + 1 - return str(var_name) + def new_var_name(self) -> str: + variable_name = chr(self.variable_index) + self.variable_index += 1 + return str(variable_name) diff --git a/numba_rvsdg/tests/test_byteflow.py b/numba_rvsdg/tests/test_byteflow.py index 65be8d5..aff5aba 100644 --- a/numba_rvsdg/tests/test_byteflow.py +++ b/numba_rvsdg/tests/test_byteflow.py @@ -113,7 +113,7 @@ def test(self): class TestPythonBytecodeBlock(unittest.TestCase): def test_constructor(self): - name_gen = NameGenerator(index=0) + name_gen = NameGenerator() block = PythonBytecodeBlock( label=PythonBytecodeLabel(), begin=0, @@ -130,7 +130,7 @@ def test_constructor(self): def test_get_instructions(self): # If the function definition line changes, just change the variable below, rest of it will adjust as long as function remains the same func_def_line = 16 - name_gen = NameGenerator(index=0) + name_gen = NameGenerator() block = PythonBytecodeBlock( label=PythonBytecodeLabel(), begin=0, From 390bc79d225ecac52bb84a4fc62704c7c56a4287 Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 15:20:23 +0200 Subject: [PATCH 14/30] refactor find_headers_and_entries We add some documentation, fixup the typing signature, refactor the function to use return sorted lists, modify some implementation details and add comments to the function. We return sorted lists, because we have been bitten in the past by using sets and them being unordered. --- numba_rvsdg/core/datastructures/scfg.py | 44 +++++++++++++++++-------- numba_rvsdg/core/transformations.py | 10 +++--- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 5110f0f..45d4ce4 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -121,8 +121,9 @@ def __iter__(self): return list(scc(GraphWrap(self.blocks, subgraph))) def find_headers_and_entries( - self, subgraph: Set[BlockName] - ) -> Tuple[Set[BlockName], Set[BlockName]]: + self, + subgraph: set[BlockName] + ) -> Tuple[list[BlockName], list[BlockName]]: """Find entries and headers in a given subgraph. Entries are blocks outside the subgraph that have an edge pointing to @@ -131,23 +132,40 @@ def find_headers_and_entries( subgraph. Entries point to headers and headers are pointed to by entries. + Parameters + ---------- + subgraph: set of BlockName + The subgraph for which to find the headers and entries + + Returns + ------- + headers: list of BlockName + The headers for this subgraph + entries: + The entries for this subgraph + + Notes + ----- + The returned lists of headers and entries are sorted. """ outside: BlockName - entries: Set[BlockName] = set() - headers: Set[BlockName] = set() - + entries: Set[BlockName] = list() + headers: Set[BlockName] = list() + # Iterate over all blocks in the graph, excluding any blocks inside the + # subgraph. for outside in self.exclude_blocks(subgraph): - nodes_jump_in_loop = subgraph.intersection( - self.out_edges[outside] - ) - headers.update(nodes_jump_in_loop) - if nodes_jump_in_loop: - entries.add(outside) + # Check if the current block points to any blocks that are inside + # the subgraph. + targets_in_loop = subgraph.intersection(self.out_edges[outside]) + # Record both headers and entries + if targets_in_loop: + headers.extend(targets_in_loop) + entries.append(outside) # If the loop has no headers or entries, the only header is the head of # the CFG. if not headers: - headers = {self.find_head()} - return headers, entries + headers = headers.append(self.find_head()) + return sorted(headers), sorted(entries) def find_exiting_and_exits( self, subgraph: Set[BlockName] diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 56f5bdd..13bf9b7 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -51,12 +51,12 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): solo_head_label = SyntheticHead() loop_head: BlockName = insert_block_and_control_blocks( scfg, - list(sorted(entries)), - list(sorted(headers)), + entries, + headers, block_label=solo_head_label) loop.add(loop_head) else: - loop_head: BlockName = next(iter(headers)) + loop_head: BlockName = headers[0] # If there is only a single exiting latch (an exiting block that also has a # backedge to the loop header) we can exit early, since the condition for # SCFG is fullfilled. @@ -314,8 +314,8 @@ def extract_region(scfg: SCFG, region_blocks, region_kind): exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(region_blocks) assert len(headers) == 1 assert len(exiting_blocks) == 1 - region_header = next(iter(headers)) - region_exiting = next(iter(exiting_blocks)) + region_header = headers[0] + region_exiting = exiting_blocks[0] scfg.add_region(region_header, region_exiting, region_kind) From 8e0dd76c07ef2ac35b24337d021e42f3293f7d07 Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 18:33:11 +0200 Subject: [PATCH 15/30] cross check trace against existing blocks As title --- numba_rvsdg/tests/simulator.py | 6 ++--- numba_rvsdg/tests/test_simulate.py | 36 ++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/numba_rvsdg/tests/simulator.py b/numba_rvsdg/tests/simulator.py index 7588411..5b0d897 100644 --- a/numba_rvsdg/tests/simulator.py +++ b/numba_rvsdg/tests/simulator.py @@ -52,7 +52,7 @@ class Simulator: Control variable map stack: List[Instruction] Instruction stack - trace: List[Tuple(name, block)] + trace: Set[BlockName] List of names, block combinations visisted branch: Boolean Flag to be set during execution. @@ -71,7 +71,7 @@ def __init__(self, flow: ByteFlow, globals: dict): self.varmap = dict() self.ctrl_varmap = dict() self.stack = [] - self.trace = [] + self.trace = set() self.branch = None self.return_value = None @@ -137,7 +137,7 @@ def run_BasicBlock(self, name: BlockName): """ print("AT", name) block = self.get_block(name) - self.trace.append((name, block)) + self.trace.add(name) if isinstance(block.label, ControlLabel): self.run_synth_block(name) diff --git a/numba_rvsdg/tests/test_simulate.py b/numba_rvsdg/tests/test_simulate.py index 9b87479..d8ad60b 100644 --- a/numba_rvsdg/tests/test_simulate.py +++ b/numba_rvsdg/tests/test_simulate.py @@ -4,10 +4,22 @@ class SimulatorTest(unittest.TestCase): + + def setUp(self): + """Initialize simulator. """ + self.sim = None + def _run(self, func, flow, kwargs): + """Run function func. """ + # lazily initialize the simulator + if self.sim is None: + self.sim = Simulator(flow, func.__globals__) + with self.subTest(): - sim = Simulator(flow, func.__globals__) - self.assertEqual(sim.run(kwargs), func(**kwargs)) + self.assertEqual(self.sim.run(kwargs), func(**kwargs)) + + def _check_trace(self, flow): + self.assertEqual(self.sim.trace, set(flow.scfg.blocks.keys())) def test_simple_branch(self): def foo(x): @@ -26,6 +38,9 @@ def foo(x): # else case self._run(foo, flow, {"x": 0}) + # check the trace + self._check_trace(flow) + def test_simple_for_loop(self): def foo(x): c = 0 @@ -43,6 +58,9 @@ def foo(x): # extended loop case self._run(foo, flow, {"x": 100}) + # check the trace + self._check_trace(flow) + def test_simple_while_loop(self): def foo(x): c = 0 @@ -62,6 +80,9 @@ def foo(x): # extended loop case self._run(foo, flow, {"x": 100}) + # check the trace + self._check_trace(flow) + def test_for_loop_with_exit(self): def foo(x): c = 0 @@ -81,6 +102,9 @@ def foo(x): # break case self._run(foo, flow, {"x": 15}) + # check the trace + self._check_trace(flow) + def test_nested_for_loop_with_break_and_continue(self): def foo(x): c = 0 @@ -107,6 +131,9 @@ def foo(x): # will break self._run(foo, flow, {"x": 5}) + # check the trace + self._check_trace(flow) + def test_for_loop_with_multiple_backedges(self): def foo(x): c = 0 @@ -131,6 +158,9 @@ def foo(x): # adding 1000, via the elif clause self._run(foo, flow, {"x": 7}) + # check the trace + self._check_trace(flow) + def test_andor(self): def foo(x, y): return (x > 0 and x < 10) or (y > 0 and y < 10) @@ -166,6 +196,8 @@ def foo(s, e): # mutiple iterations self._run(foo, flow, {"s": 23, "e": 28}) + # check the trace + self._check_trace(flow) if __name__ == "__main__": unittest.main() From a5cfba2694b140153335704b675bde660d1923bb Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 Mar 2023 18:46:58 +0200 Subject: [PATCH 16/30] fix simulator test bug found by tracing As title --- numba_rvsdg/tests/test_simulate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numba_rvsdg/tests/test_simulate.py b/numba_rvsdg/tests/test_simulate.py index d8ad60b..b13219b 100644 --- a/numba_rvsdg/tests/test_simulate.py +++ b/numba_rvsdg/tests/test_simulate.py @@ -100,7 +100,7 @@ def foo(x): # loop case self._run(foo, flow, {"x": 2}) # break case - self._run(foo, flow, {"x": 15}) + self._run(foo, flow, {"x": 101}) # check the trace self._check_trace(flow) From 129b6e0788526210f0eff0f345fd8f4710647bf8 Mon Sep 17 00:00:00 2001 From: esc Date: Thu, 30 Mar 2023 15:50:20 +0200 Subject: [PATCH 17/30] fixup formatting and typos in README As title --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 70dfcad..4067be3 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ within Numba. The code in this repository is an implementation of the CFG restructuring algorithms in Bahmann2015, specifically those from section 4.1 and 4.2: namely "loop restructuring" and "branch restructuring". These are interesting for -Numba because they serve to clearly identify regions withing the Python +Numba because they serve to clearly identify regions within the Python bytecode. ## dependencies @@ -42,7 +42,7 @@ numba_rvsdg ├── core │   ├── datastructures │   │   ├── basic_block.py # BasicBlock implementation -│   │   ├── scfg.py # SCFG implementation, maps labels to blocks +│   │   ├── scfg.py # SCFG implementation, maps labels to blocks │   │   ├── byte_flow.py # ByteFlow implementation, SCFG + bytecode │   │   ├── flow_info.py # Converts program to ByteFlow │   │   └── labels.py # Collection of Label classes From 58054c358bab1c0c9097790e4ea997ce2614ef24 Mon Sep 17 00:00:00 2001 From: esc Date: Fri, 31 Mar 2023 09:19:32 +0200 Subject: [PATCH 18/30] fix two bugs leftover from refactoring This fixes two bugs on `main` introduced by: * https://github.com/numba/numba-rvsdg/pull/26 and * https://github.com/numba/numba-rvsdg/pull/28 It's probably a sign that the test coverage isn't high enough yet. --- numba_rvsdg/core/transformations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 13bf9b7..6e2e6d5 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -268,9 +268,9 @@ def find_branch_regions(scfg: SCFG, begin: BlockName, end: BlockName) -> Set[Blo branch_regions = [] out_targets = scfg.out_edges[begin] for bra_start in out_targets: - for out_targets in jump_targets: + for out_target in out_targets: if (out_targets != bra_start - and scfg.is_reachable_dfs(out_targets, bra_start)): + and scfg.is_reachable_dfs(out_target, bra_start)): branch_regions.append(tuple()) break else: @@ -315,7 +315,7 @@ def extract_region(scfg: SCFG, region_blocks, region_kind): assert len(headers) == 1 assert len(exiting_blocks) == 1 region_header = headers[0] - region_exiting = exiting_blocks[0] + region_exiting = next(iter(exiting_blocks)) scfg.add_region(region_header, region_exiting, region_kind) From 4b28dc553f2791c11d9c233a8fa6f1442f83a871 Mon Sep 17 00:00:00 2001 From: esc Date: Fri, 31 Mar 2023 15:48:58 +0200 Subject: [PATCH 19/30] refactor find_exiting_and_exits Adding documentation, slight internal refactor, change return signature to be sorted lists. Also, refactored all uses of the function to account for the change in return signature. --- numba_rvsdg/core/datastructures/scfg.py | 34 +++++++++++++++++++------ numba_rvsdg/core/transformations.py | 13 +++++----- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 45d4ce4..5722141 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -169,28 +169,46 @@ def find_headers_and_entries( def find_exiting_and_exits( self, subgraph: Set[BlockName] - ) -> Tuple[Set[BlockName], Set[BlockName]]: + ) -> Tuple[list[BlockName], list[BlockName]]: """Find exiting and exit blocks in a given subgraph. - Existing blocks are blocks inside the subgraph that have edges to + Exiting blocks are blocks inside the subgraph that have edges to blocks outside of the subgraph. Exit blocks are blocks outside the subgraph that have incoming edges from within the subgraph. Exiting blocks point to exits and exits and pointed to by exiting blocks. + Parameters + ---------- + subgraph: set of BlockName + The subgraph for which to find the exiting and exit blocks. + + Returns + ------- + exiting: list of BlockName + The exiting blocks for this subgraph + exits: + The exit block for this subgraph + + Notes + ----- + The returned lists of exiting and exit blocks are sorted. + """ inside: BlockName - exiting: Set[BlockName] = set() - exits: Set[BlockName] = set() + # use sets internally to avoid duplicates + exiting: set[BlockName] = set() + exits: set[BlockName] = set() for inside in subgraph: # any node inside that points outside the loop - for jt in self.out_edges[inside]: - if jt not in subgraph: + for out_target in self.out_edges[inside]: + if out_target not in subgraph: exiting.add(inside) - exits.add(jt) + exits.add(out_target) # any returns if self.is_exiting(inside): exiting.add(inside) - return exiting, exits + # convert to sorted list before return + return sorted(exiting), sorted(exits) def is_reachable_dfs(self, begin: BlockName, end: BlockName): # -> TypeGuard: """Is end reachable from begin.""" diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 6e2e6d5..5bfb695 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -42,7 +42,6 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): headers, entries = scfg.find_headers_and_entries(loop) exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(loop) headers_were_unified = False - exit_blocks = list(sorted(exit_blocks)) # If there are multiple headers, insert assignment and control blocks, # such that only a single loop header remains. @@ -67,7 +66,7 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): if ( len(backedge_blocks) == 1 and len(exiting_blocks) == 1 - and backedge_blocks[0] == next(iter(exiting_blocks)) + and backedge_blocks[0] == exiting_blocks[0] ): scfg.back_edges[backedge_blocks[0]].append(loop_head) return @@ -106,7 +105,7 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): ) else: backedge_value_table = dict( - (i, j) for i, j in enumerate((loop_head, next(iter(exit_blocks)))) + (i, j) for i, j in enumerate((loop_head, exit_blocks[0])) ) if headers_were_unified: header_value_table = scfg[loop_head].branch_value_table @@ -155,7 +154,7 @@ def reverse_lookup(d, value): ) variable_assignment[backedge_variable] = reverse_lookup( backedge_value_table, - synth_exit if needs_synth_exit else next(iter(exit_blocks)), + synth_exit if needs_synth_exit else exit_blocks[0] ) # Insert the assignment to the block map synth_assign = scfg.add_block( @@ -213,9 +212,9 @@ def reverse_lookup(d, value): scfg.insert_block_between( synth_exit, [synth_exiting_latch], - list(exit_blocks)) + exit_blocks) else: - scfg.out_edges[synth_exiting_latch].append(list(exit_blocks)[0]) + scfg.out_edges[synth_exiting_latch].append(exit_blocks[0]) def restructure_loop(scfg: SCFG): @@ -315,7 +314,7 @@ def extract_region(scfg: SCFG, region_blocks, region_kind): assert len(headers) == 1 assert len(exiting_blocks) == 1 region_header = headers[0] - region_exiting = next(iter(exiting_blocks)) + region_exiting = exiting_blocks[0] scfg.add_region(region_header, region_exiting, region_kind) From 6c7e045bea4cc1375cefd32aad6908c4f49b15e5 Mon Sep 17 00:00:00 2001 From: kc611 Date: Wed, 29 Mar 2023 18:52:38 +0530 Subject: [PATCH 20/30] Added region heirarchy and fixed region rendering --- numba_rvsdg/core/datastructures/labels.py | 19 ++-- numba_rvsdg/core/datastructures/region.py | 29 +++++++ numba_rvsdg/core/datastructures/scfg.py | 23 +++-- numba_rvsdg/core/transformations.py | 2 +- numba_rvsdg/rendering/rendering.py | 101 ++++++++++++++-------- numba_rvsdg/tests/test_scfg.py | 4 +- 6 files changed, 125 insertions(+), 53 deletions(-) diff --git a/numba_rvsdg/core/datastructures/labels.py b/numba_rvsdg/core/datastructures/labels.py index 6aadb6c..94e802c 100644 --- a/numba_rvsdg/core/datastructures/labels.py +++ b/numba_rvsdg/core/datastructures/labels.py @@ -83,15 +83,24 @@ def get_label_class(label_type_string): @dataclass(frozen=True, order=True) -class BlockName: +class Name: name: str - ... + + def __repr__(self): + return self.name + + def __str__(self): + return self.name @dataclass(frozen=True, order=True) -class RegionName: - name: str - ... +class BlockName(Name): + pass + + +@dataclass(frozen=True, order=True) +class RegionName(Name): + pass @dataclass diff --git a/numba_rvsdg/core/datastructures/region.py b/numba_rvsdg/core/datastructures/region.py index 5c4c0c8..cfe1176 100644 --- a/numba_rvsdg/core/datastructures/region.py +++ b/numba_rvsdg/core/datastructures/region.py @@ -16,7 +16,36 @@ class Region: kind: str header: BlockName exiting: BlockName + scfg: "SCFG" + sub_region_headers: dict[BlockName, list[RegionName]] = field(default_factory=dict, init=False) def __post_init__(self, name_gen): region_name = name_gen.new_region_name(kind=self.kind) object.__setattr__(self, "region_name", region_name) + + def __iter__(self): + """Graph Iterator""" + # initialise housekeeping datastructures + to_visit, seen = [self.header], [] + while to_visit: + # get the next block_name on the list + block_name = to_visit.pop(0) + # if we have visited this, we skip it + if block_name in seen: + continue + else: + seen.append(block_name) + # yield the block_name + yield block_name + if block_name is self.exiting: + continue + # finally add any out_edges to the list of block_names to visit + to_visit.extend(self.scfg.out_edges[block_name]) + + @property + def is_leaf_region(self): + return len(self.subregions) == 0 + + @property + def is_root_region(self): + return self in self.scfg.region_roots diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 5722141..e4d9398 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -22,13 +22,15 @@ class SCFG: And stores the jump targets and back edges for blocks within the graph.""" - blocks: Dict[BlockName, BasicBlock] = field(default_factory=dict) + blocks: Dict[BlockName, BasicBlock] = field(default_factory=dict, init=False) - out_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict) - back_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict) - regions: Dict[RegionName, Region] = field(default_factory=dict) + out_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict, init=False) + back_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict, init=False) - name_gen: NameGenerator = field(default_factory=NameGenerator, compare=False) + regions: Dict[RegionName, Region] = field(default_factory=dict, init=False) + region_headers: Dict[BlockName, List[RegionName]] = field(default_factory=dict, init=False) + + name_gen: NameGenerator = field(default_factory=NameGenerator, compare=False, init=False) def __getitem__(self, index: BlockName) -> BasicBlock: return self.blocks[index] @@ -51,7 +53,7 @@ def __iter__(self): # get the corresponding block for the block_name block = self[block_name] # yield the block_name, block combo - yield (block_name, block) + yield block_name # finally add any out_edges to the list of block_names to visit to_visit.extend(self.out_edges[block_name]) @@ -296,8 +298,13 @@ def add_connections(self, block_name, out_edges=[], back_edges=[]): self.check_graph() def add_region(self, region_head, region_exit, kind): - new_region = Region(self.name_gen, kind, region_head, region_exit) - self.regions[new_region.region_name] = new_region + new_region = Region(self.name_gen, kind, region_head, region_exit, self) + region_name = new_region.region_name + self.regions[region_name] = new_region + if region_head in self.region_headers: + self.region_headers[region_head].append(region_name) + else: + self.region_headers[region_head] = [region_name] @staticmethod def from_yaml(yaml_string): diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index 5bfb695..ee56a65 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -268,7 +268,7 @@ def find_branch_regions(scfg: SCFG, begin: BlockName, end: BlockName) -> Set[Blo out_targets = scfg.out_edges[begin] for bra_start in out_targets: for out_target in out_targets: - if (out_targets != bra_start + if (out_target != bra_start and scfg.is_reachable_dfs(out_target, bra_start)): branch_regions.append(tuple()) break diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 1f15f63..e2c1856 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -26,26 +26,12 @@ def __init__(self, byte_flow: ByteFlow): self.scfg = byte_flow.scfg self.bcmap_from_bytecode(byte_flow.bc) - self.render_blocks() + self.rendered_blocks = set() + self.rendered_regions = set() + self.render_region(self.g, None) self.render_edges() - self.render_regions() - - def render_regions(self): - # render subgraph - # for region_name, region in self.scfg.regions.items(): - # graph = region.get_full_graph() - # with self.g.subgraph(name=f"cluster_{label}") as subg: - # color = "blue" - # if region.kind == "branch": - # color = "green" - # if region.kind == "tail": - # color = "purple" - # if region.kind == "head": - # color = "red" - # subg.attr(color=color, label=region.kind) - pass - - def render_basic_block(self, block_name: BlockName): + + def render_basic_block(self, graph, block_name: BlockName): block = self.scfg[block_name] if isinstance(block.label, PythonBytecodeLabel): @@ -58,9 +44,9 @@ def render_basic_block(self, block_name: BlockName): body = str(block_name) else: raise Exception("Unknown label type: " + block.label) - self.g.node(str(block_name), shape="rect", label=body) + graph.node(str(block_name), shape="rect", label=body) - def render_control_variable_block(self, block_name: BlockName): + def render_control_variable_block(self, graph, block_name: BlockName): block = self.scfg[block_name] if isinstance(block.label, ControlLabel): @@ -70,9 +56,9 @@ def render_control_variable_block(self, block_name: BlockName): # ) else: raise Exception("Unknown label type: " + block.label) - self.g.node(str(block_name), shape="rect", label=body) + graph.node(str(block_name), shape="rect", label=body) - def render_branching_block(self, block_name: BlockName): + def render_branching_block(self, graph, block_name: BlockName): block = self.scfg[block_name] if isinstance(block.label, ControlLabel): @@ -90,20 +76,61 @@ def find_index(v): # ) else: raise Exception("Unknown label type: " + block.label) - self.g.node(str(block_name), shape="rect", label=body) - - def render_blocks(self): - for block_name, block in self.scfg.blocks.items(): - if type(block) == BasicBlock: - self.render_basic_block(block_name) - elif type(block) == PythonBytecodeBlock: - self.render_basic_block(block_name) - elif type(block) == ControlVariableBlock: - self.render_control_variable_block(block_name) - elif type(block) == BranchBlock: - self.render_branching_block(block_name) - else: - raise Exception("unreachable") + graph.node(str(block_name), shape="rect", label=body) + + def render_region(self, graph, region_name): + # If region name is none, we're in the 'root' region + # that is the graph itself. + if region_name is None: + region_headers = self.scfg.region_headers + all_blocks = list(self.scfg) + else: + region = self.scfg.regions[region_name] + region_headers = region.sub_region_headers + all_blocks = list(region) + + + # If subregions exist within this region we render them first + if region_headers: + for _, regions in region_headers.items(): + for _region_name in regions: + _region = self.scfg.regions[_region_name] + with graph.subgraph(name=f"cluster_{_region_name}") as subg: + color = "blue" + if _region.kind == "branch": + color = "green" + if _region.kind == "tail": + color = "purple" + if _region.kind == "head": + color = "red" + subg.attr(color=color, label=_region.kind) + self.render_region(subg, _region_name) + + # If there are no further subregions then we render the blocks + for block_name in all_blocks: + self.render_block(graph, block_name, self.scfg[block_name]) + + def render_block(self, graph, block_name, block): + if block_name in self.rendered_blocks: + return + + if block_name in self.scfg.region_headers.keys(): + for region in self.scfg.region_headers[block_name]: + if region not in self.rendered_regions: + self.rendered_regions.add(region) + self.render_region(graph, region) + + if type(block) == BasicBlock: + self.render_basic_block(graph, block_name) + elif type(block) == PythonBytecodeBlock: + self.render_basic_block(graph, block_name) + elif type(block) == ControlVariableBlock: + self.render_control_variable_block(graph, block_name) + elif type(block) == BranchBlock: + self.render_branching_block(graph, block_name) + else: + raise Exception("unreachable") + self.rendered_blocks.add(block_name) def render_edges(self): diff --git a/numba_rvsdg/tests/test_scfg.py b/numba_rvsdg/tests/test_scfg.py index 1717121..104bded 100644 --- a/numba_rvsdg/tests/test_scfg.py +++ b/numba_rvsdg/tests/test_scfg.py @@ -115,8 +115,8 @@ def test_scfg_iter(self): block_0 = BasicBlock(name_generator, Label()) block_1 = BasicBlock(name_generator, Label()) expected = [ - (block_0.block_name, block_0), - (block_1.block_name, block_1), + block_0.block_name, + block_1.block_name, ] scfg, ref_dict = SCFG.from_yaml( """ From b3e71aabbdedf1771f278ce114b3391f23c6f6b9 Mon Sep 17 00:00:00 2001 From: kc611 Date: Wed, 29 Mar 2023 19:06:57 +0530 Subject: [PATCH 21/30] Fixed failing tests regarding string representation --- numba_rvsdg/core/datastructures/scfg.py | 4 ++-- numba_rvsdg/tests/test_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index e4d9398..b203a8d 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -343,7 +343,7 @@ def to_yaml(self): for key, value in self.blocks.items(): out_edges = [f"{i}" for i in self.out_edges[key]] - # out_edges = str(out_edges).replace("'", '"') + out_edges = str(out_edges).replace("'", '"') back_edges = [f"{i}" for i in self.back_edges[key]] jump_target_str = f""" "{str(key)}": @@ -351,7 +351,7 @@ def to_yaml(self): out: {out_edges}""" if back_edges: - # back_edges = str(back_edges).replace("'", '"') + back_edges = str(back_edges).replace("'", '"') jump_target_str += f""" back: {back_edges}""" yaml_string += dedent(jump_target_str) diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index 97aee1f..7020156 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -22,7 +22,7 @@ def assertSCFGEqual(self, first_scfg: SCFG, second_scfg: SCFG): def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): for key, value in ref_dict.items(): - second_yaml = second_yaml.replace(repr(value), key) + first_yaml = first_yaml.replace(key, value.name) self.assertEqual(first_yaml, second_yaml) From 75575261bfe0ba143d175868e80352e9297718ff Mon Sep 17 00:00:00 2001 From: kc611 Date: Wed, 29 Mar 2023 19:16:12 +0530 Subject: [PATCH 22/30] Removed scfg from regions --- numba_rvsdg/core/datastructures/region.py | 28 +---------------------- numba_rvsdg/core/datastructures/scfg.py | 22 +++++++++++++++++- numba_rvsdg/rendering/rendering.py | 7 +++--- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/numba_rvsdg/core/datastructures/region.py b/numba_rvsdg/core/datastructures/region.py index cfe1176..5e12b74 100644 --- a/numba_rvsdg/core/datastructures/region.py +++ b/numba_rvsdg/core/datastructures/region.py @@ -16,36 +16,10 @@ class Region: kind: str header: BlockName exiting: BlockName - scfg: "SCFG" + sub_region_headers: dict[BlockName, list[RegionName]] = field(default_factory=dict, init=False) def __post_init__(self, name_gen): region_name = name_gen.new_region_name(kind=self.kind) object.__setattr__(self, "region_name", region_name) - def __iter__(self): - """Graph Iterator""" - # initialise housekeeping datastructures - to_visit, seen = [self.header], [] - while to_visit: - # get the next block_name on the list - block_name = to_visit.pop(0) - # if we have visited this, we skip it - if block_name in seen: - continue - else: - seen.append(block_name) - # yield the block_name - yield block_name - if block_name is self.exiting: - continue - # finally add any out_edges to the list of block_names to visit - to_visit.extend(self.scfg.out_edges[block_name]) - - @property - def is_leaf_region(self): - return len(self.subregions) == 0 - - @property - def is_root_region(self): - return self in self.scfg.region_roots diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index b203a8d..527388d 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -298,7 +298,7 @@ def add_connections(self, block_name, out_edges=[], back_edges=[]): self.check_graph() def add_region(self, region_head, region_exit, kind): - new_region = Region(self.name_gen, kind, region_head, region_exit, self) + new_region = Region(self.name_gen, kind, region_head, region_exit) region_name = new_region.region_name self.regions[region_name] = new_region if region_head in self.region_headers: @@ -370,3 +370,23 @@ def to_dict(self): graph_dict[str(key)] = curr_dict return graph_dict + + def iterate_region(self, region_name): + region = self.regions[region_name] + """Region Iterator""" + # initialise housekeeping datastructures + to_visit, seen = [region.header], [] + while to_visit: + # get the next block_name on the list + block_name = to_visit.pop(0) + # if we have visited this, we skip it + if block_name in seen: + continue + else: + seen.append(block_name) + # yield the block_name + yield block_name + if block_name is region.exiting: + continue + # finally add any out_edges to the list of block_names to visit + to_visit.extend(self.out_edges[block_name]) diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index e2c1856..2bd0c2a 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -83,12 +83,11 @@ def render_region(self, graph, region_name): # that is the graph itself. if region_name is None: region_headers = self.scfg.region_headers - all_blocks = list(self.scfg) + all_blocks_iterator = self.scfg else: region = self.scfg.regions[region_name] region_headers = region.sub_region_headers - all_blocks = list(region) - + all_blocks_iterator = self.scfg.iterate_region(region_name) # If subregions exist within this region we render them first if region_headers: @@ -107,7 +106,7 @@ def render_region(self, graph, region_name): self.render_region(subg, _region_name) # If there are no further subregions then we render the blocks - for block_name in all_blocks: + for block_name in all_blocks_iterator: self.render_block(graph, block_name, self.scfg[block_name]) def render_block(self, graph, block_name, block): From f184a8102702ac36e1ee8f234d3b0fe294f612e2 Mon Sep 17 00:00:00 2001 From: kc611 Date: Fri, 31 Mar 2023 16:34:37 +0530 Subject: [PATCH 23/30] Added dummy Region within SCFG graphs --- .../core/datastructures/basic_block.py | 16 ++- numba_rvsdg/core/datastructures/flow_info.py | 2 +- numba_rvsdg/core/datastructures/labels.py | 19 ++- numba_rvsdg/core/datastructures/region.py | 46 ++++-- numba_rvsdg/core/datastructures/scfg.py | 136 +++++++++++------- numba_rvsdg/core/transformations.py | 15 +- numba_rvsdg/rendering/rendering.py | 79 +++++----- numba_rvsdg/tests/test_utils.py | 2 +- 8 files changed, 191 insertions(+), 124 deletions(-) diff --git a/numba_rvsdg/core/datastructures/basic_block.py b/numba_rvsdg/core/datastructures/basic_block.py index 6a703a3..e62cd6e 100644 --- a/numba_rvsdg/core/datastructures/basic_block.py +++ b/numba_rvsdg/core/datastructures/basic_block.py @@ -7,19 +7,23 @@ @dataclass(frozen=True) -class BasicBlock: +class Block: name_gen: InitVar[NameGenerator] - """Block Name Generator associated with this BasicBlock. + """Block Name Generator associated with this Block. Note: This is an initialization only argument and not a class attribute.""" - block_name: BlockName = field(init=False) - """Unique name identifier for this block""" - label: Label """The corresponding Label for this block.""" - def __post_init__(self, name_gen): + +@dataclass(frozen=True) +class BasicBlock(Block): + + block_name: BlockName = field(init=False) + """Unique name identifier for this block""" + + def __post_init__(self, name_gen: NameGenerator): block_name = name_gen.new_block_name(label=self.label) object.__setattr__(self, "block_name", block_name) diff --git a/numba_rvsdg/core/datastructures/flow_info.py b/numba_rvsdg/core/datastructures/flow_info.py index 528ba62..f0e8a18 100644 --- a/numba_rvsdg/core/datastructures/flow_info.py +++ b/numba_rvsdg/core/datastructures/flow_info.py @@ -94,7 +94,7 @@ def build_basicblocks(self: "FlowInfo", end_offset=None) -> "SCFG": targets = [names[o] for o in self.jump_insts[term_offset]] block_name = names[begin] - scfg.add_connections(block_name, targets, []) + scfg.add_connections(block_name, targets) scfg.check_graph() return scfg diff --git a/numba_rvsdg/core/datastructures/labels.py b/numba_rvsdg/core/datastructures/labels.py index 94e802c..98c4393 100644 --- a/numba_rvsdg/core/datastructures/labels.py +++ b/numba_rvsdg/core/datastructures/labels.py @@ -19,6 +19,11 @@ class ControlLabel(Label): pass +@dataclass(frozen=True, order=True) +class RegionLabel(Label): + pass + + @dataclass(frozen=True, order=True) class SyntheticBranch(ControlLabel): pass @@ -59,6 +64,16 @@ class SynthenticAssignment(ControlLabel): pass +@dataclass(frozen=True, order=True) +class LoopRegionLabel(RegionLabel): + pass + + +@dataclass(frozen=True, order=True) +class MetaRegionLabel(RegionLabel): + pass + + # Maybe we can register new labels over here instead of static lists label_types = { "label": Label, @@ -126,10 +141,10 @@ def new_block_name(self, label: str) -> BlockName: self.block_index += 1 return BlockName(str(label).lower().split("(")[0] + "_" + str(ret)) - def new_region_name(self, kind: str) -> RegionName: + def new_region_name(self, label: str) -> RegionName: ret = self.region_index self.region_index += 1 - return RegionName(str(kind).lower().split("(")[0] + "_" + str(ret)) + return RegionName(str(label).lower().split("(")[0] + "_" + str(ret)) def new_var_name(self) -> str: variable_name = chr(self.variable_index) diff --git a/numba_rvsdg/core/datastructures/region.py b/numba_rvsdg/core/datastructures/region.py index 5e12b74..114802b 100644 --- a/numba_rvsdg/core/datastructures/region.py +++ b/numba_rvsdg/core/datastructures/region.py @@ -1,25 +1,47 @@ -from dataclasses import dataclass, field, InitVar +from dataclasses import dataclass, field +from numba_rvsdg.core.datastructures.basic_block import Block from numba_rvsdg.core.datastructures.labels import NameGenerator, RegionName, BlockName @dataclass(frozen=True) -class Region: - name_gen: InitVar[NameGenerator] - """Region Name Generator associated with this Region. - Note: This is an initialization only argument and not - a class attribute.""" - +class Region(Block): region_name: RegionName = field(init=False) """Unique name identifier for this region""" - kind: str + def __post_init__(self, name_gen: NameGenerator): + region_name = name_gen.new_region_name(self.label) + object.__setattr__(self, "region_name", region_name) + + +@dataclass(frozen=True) +class LoopRegion(Region): header: BlockName exiting: BlockName + ... - sub_region_headers: dict[BlockName, list[RegionName]] = field(default_factory=dict, init=False) - def __post_init__(self, name_gen): - region_name = name_gen.new_region_name(kind=self.kind) - object.__setattr__(self, "region_name", region_name) +@dataclass(frozen=True) +class MetaRegion(Region): + ... + + +# TODO: Register new regions over here +region_types = { + "loop": LoopRegion +} + + +def get_region_class(region_type_string: str): + if region_type_string in region_types: + return region_types[region_type_string] + else: + raise TypeError(f"Region Type {region_type_string} not recognized.") + +def get_region_class_str(region: Region): + for key, value in region_types.items(): + if isinstance(region, value): + return key + else: + raise TypeError(f"Region Type of {region} not recognized.") diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 527388d..e7add0d 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -6,12 +6,13 @@ from dataclasses import dataclass, field from numba_rvsdg.core.datastructures.basic_block import BasicBlock, get_block_class, get_block_class_str -from numba_rvsdg.core.datastructures.region import Region +from numba_rvsdg.core.datastructures.region import MetaRegion, Region, LoopRegion, get_region_class from numba_rvsdg.core.datastructures.labels import ( Label, BlockName, NameGenerator, RegionName, + MetaRegionLabel, LoopRegionLabel, RegionLabel, get_label_class, ) @@ -25,13 +26,21 @@ class SCFG: blocks: Dict[BlockName, BasicBlock] = field(default_factory=dict, init=False) out_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict, init=False) - back_edges: Dict[BlockName, List[BlockName]] = field(default_factory=dict, init=False) + back_edges: set[tuple[BlockName, BlockName]] = field(default_factory=set, init=False) regions: Dict[RegionName, Region] = field(default_factory=dict, init=False) - region_headers: Dict[BlockName, List[RegionName]] = field(default_factory=dict, init=False) + meta_region: RegionName = field(init=False) + region_tree: Dict[BlockName, List[RegionName]] = field(default_factory=dict, init=False) name_gen: NameGenerator = field(default_factory=NameGenerator, compare=False, init=False) + def __post_init__(self): + new_region = MetaRegion(name_gen = self.name_gen, label = MetaRegionLabel()) + region_name = new_region.region_name + self.regions[region_name] = new_region + self.region_tree[region_name] = [] + object.__setattr__(self, "meta_region", region_name) + def __getitem__(self, index: BlockName) -> BasicBlock: return self.blocks[index] @@ -43,19 +52,20 @@ def __iter__(self): # initialise housekeeping datastructures to_visit, seen = [self.find_head()], [] while to_visit: - # get the next block_name on the list - block_name = to_visit.pop(0) + # get the next name on the list + name = to_visit.pop(0) # if we have visited this, we skip it - if block_name in seen: + if name in seen: continue else: - seen.append(block_name) - # get the corresponding block for the block_name - block = self[block_name] - # yield the block_name, block combo - yield block_name - # finally add any out_edges to the list of block_names to visit - to_visit.extend(self.out_edges[block_name]) + seen.append(name) + # get the corresponding block for the name + if isinstance(name, RegionName): + name = self.regions[name].header + # yield the name, block combo + yield name + # finally add any out_edges to the list of names to visit + to_visit.extend(self.out_edges[name]) def exclude_blocks(self, exclude_blocks: Set[BlockName]) -> Iterator[BlockName]: """Iterator over all nodes not in exclude_blocks.""" @@ -74,6 +84,9 @@ def find_head(self) -> BlockName: for name in self.blocks.keys(): for jt in self.out_edges[name]: heads.discard(jt) + for _, region in self.regions.items(): + if hasattr(region, "header"): + heads.discard(region.header) assert len(heads) == 1 return next(iter(heads)) @@ -151,8 +164,8 @@ def find_headers_and_entries( The returned lists of headers and entries are sorted. """ outside: BlockName - entries: Set[BlockName] = list() - headers: Set[BlockName] = list() + entries: set[BlockName] = set() + headers: set[BlockName] = set() # Iterate over all blocks in the graph, excluding any blocks inside the # subgraph. for outside in self.exclude_blocks(subgraph): @@ -161,13 +174,13 @@ def find_headers_and_entries( targets_in_loop = subgraph.intersection(self.out_edges[outside]) # Record both headers and entries if targets_in_loop: - headers.extend(targets_in_loop) - entries.append(outside) + headers.update(targets_in_loop) + entries.add(outside) # If the loop has no headers or entries, the only header is the head of # the CFG. if not headers: - headers = headers.append(self.find_head()) - return sorted(headers), sorted(entries) + headers.add(self.find_head()) + return sorted(list(headers)), sorted(list(entries)) def find_exiting_and_exits( self, subgraph: Set[BlockName] @@ -240,14 +253,6 @@ def is_fallthrough(self, block_name: BlockName): def check_graph(self): pass - # We don't need this cause everything is 'hopefully' additive - # def remove_blocks(self, names: Set[BlockName]): - # for name in names: - # del self.blocks[name] - # del self.out_edges[name] - # del self.back_edges[name] - # self.check_graph() - def insert_block_between( self, block_name: BlockName, @@ -283,28 +288,35 @@ def add_block( name = new_block.block_name self.blocks[name] = new_block - - self.back_edges[name] = [] self.out_edges[name] = [] return name - def add_connections(self, block_name, out_edges=[], back_edges=[]): + def add_region(self, kind: str, header: BlockName, exiting: BlockName, parent: Region = None, region_label = RegionLabel()): + if parent is None: + parent = self.meta_region + + region_type = get_region_class(kind) + new_region: Region = region_type(name_gen=self.name_gen, label=region_label, header=header, exiting=exiting) + region_name = new_region.region_name + self.regions[region_name] = new_region + self.region_tree[region_name] = [] + + self.region_tree[parent].append(region_name) + + for block, out_edges in self.out_edges.items(): + for idx, edge in enumerate(out_edges): + if edge == header and block is not exiting: + self.out_edges[block][idx] = region_name + + return region_name + + def add_connections(self, block_name, out_edges=[]): assert self.out_edges[block_name] == [] - assert self.back_edges[block_name] == [] self.out_edges[block_name] = out_edges - self.back_edges[block_name] = back_edges self.check_graph() - def add_region(self, region_head, region_exit, kind): - new_region = Region(self.name_gen, kind, region_head, region_exit) - region_name = new_region.region_name - self.regions[region_name] = new_region - if region_head in self.region_headers: - self.region_headers[region_head].append(region_name) - else: - self.region_headers[region_head] = [region_name] @staticmethod def from_yaml(yaml_string): @@ -331,8 +343,10 @@ def from_dict(graph_dict: Dict[str, Dict]): block_name = ref_dict[block_ref] out_edges = list(ref_dict[out_ref] for out_ref in out_refs) - back_edges = list(ref_dict[back_ref] for back_ref in back_refs) - scfg.add_connections(block_name, out_edges, back_edges) + scfg.add_connections(block_name, out_edges) + for _back in back_refs: + scfg.back_edges.add((ref_dict[block_ref], ref_dict[_back])) + scfg.check_graph() return scfg, ref_dict @@ -342,9 +356,13 @@ def to_yaml(self): yaml_string = """""" for key, value in self.blocks.items(): - out_edges = [f"{i}" for i in self.out_edges[key]] + out_edges = [] + back_edges = [] + for out_edge in self.out_edges[key]: + out_edges.append(f"{out_edge}") + if (key, out_edge) in self.back_edges: + back_edges.append(f"{out_edge}") out_edges = str(out_edges).replace("'", '"') - back_edges = [f"{i}" for i in self.back_edges[key]] jump_target_str = f""" "{str(key)}": type: "{get_block_class_str(value)}" @@ -363,19 +381,28 @@ def to_dict(self): for key, value in self.blocks.items(): curr_dict = {} curr_dict["type"] = get_block_class_str(value) - curr_dict["out"] = [f"{i}" for i in self.out_edges[key]] - back_edges = [f"{i}" for i in self.back_edges[key]] + curr_dict["out"] = [] + back_edges = [] + for out_edge in self.out_edges[key]: + curr_dict["out"].append(f"{out_edge}") + if (key, out_edge) in self.back_edges: + back_edges.append(f"{out_edge}") if back_edges: curr_dict["back"] = back_edges graph_dict[str(key)] = curr_dict return graph_dict - def iterate_region(self, region_name): + def iterate_region(self, region_name, region_view=False): + if region_name == self.meta_region and not region_view: + return iter(self) + region = self.regions[region_name] """Region Iterator""" + region_head = region.header if region_name is not self.meta_region else self.find_head() + # initialise housekeeping datastructures - to_visit, seen = [region.header], [] + to_visit, seen = [region_head], [] while to_visit: # get the next block_name on the list block_name = to_visit.pop(0) @@ -386,7 +413,16 @@ def iterate_region(self, region_name): seen.append(block_name) # yield the block_name yield block_name - if block_name is region.exiting: + if region_name is not self.meta_region and block_name is region.exiting: continue # finally add any out_edges to the list of block_names to visit - to_visit.extend(self.out_edges[block_name]) + outs = self.out_edges[block_name] + + if not region_view: + for idx, _out in enumerate(outs): + if isinstance(_out, RegionName): + to_visit.append(_out.header) + else: + to_visit.append(_out) + else: + to_visit.extend(outs) diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index ee56a65..e78022c 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -12,6 +12,7 @@ SynthenticAssignment, PythonBytecodeLabel, BlockName, + RegionLabel ) from numba_rvsdg.core.datastructures.scfg import SCFG from numba_rvsdg.core.datastructures.basic_block import ( @@ -68,7 +69,7 @@ def loop_restructure_helper(scfg: SCFG, loop: Set[BlockName]): and len(exiting_blocks) == 1 and backedge_blocks[0] == exiting_blocks[0] ): - scfg.back_edges[backedge_blocks[0]].append(loop_head) + scfg.back_edges.add((backedge_blocks[0], loop_head)) return doms = _doms(scfg) @@ -163,8 +164,7 @@ def reverse_lookup(d, value): variable_assignment=variable_assignment) scfg.add_connections( synth_assign, - [synth_exiting_latch], - []) + [synth_exiting_latch]) loop.add(synth_assign) # Update the edge from the out_target to point to the new # assignment block @@ -190,8 +190,7 @@ def reverse_lookup(d, value): variable_assignment=variable_assignment) scfg.add_connections( synth_assign, - [synth_exiting_latch], - []) + [synth_exiting_latch]) loop.add(synth_assign) # Update the edge from the out_target to point to the new @@ -204,7 +203,7 @@ def reverse_lookup(d, value): # Add the back_edge scfg.out_edges[synth_exiting_latch].append(loop_head) - scfg.back_edges[synth_exiting_latch].append(loop_head) + scfg.back_edges.add((synth_exiting_latch, loop_head)) # If an exit is to be created, we do so too, but only add it to the scfg, # since it isn't part of the loop @@ -308,7 +307,7 @@ def find_tail_blocks(scfg: SCFG, begin: Set[BlockName], head_region_blocks, bran return tail_subregion -def extract_region(scfg: SCFG, region_blocks, region_kind): +def extract_region(scfg: SCFG, region_blocks, region_kind, region_label = RegionLabel()): headers, entries = scfg.find_headers_and_entries(region_blocks) exiting_blocks, exit_blocks = scfg.find_exiting_and_exits(region_blocks) assert len(headers) == 1 @@ -316,7 +315,7 @@ def extract_region(scfg: SCFG, region_blocks, region_kind): region_header = headers[0] region_exiting = exiting_blocks[0] - scfg.add_region(region_header, region_exiting, region_kind) + scfg.add_region(region_kind, region_label=region_label, header = region_header, exiting = region_exiting) def restructure_branch(scfg: SCFG): diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 2bd0c2a..2ef5cb1 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -11,7 +11,9 @@ PythonBytecodeLabel, ControlLabel, BlockName, + RegionName ) +from numba_rvsdg.core.datastructures.region import MetaRegion, LoopRegion from numba_rvsdg.core.datastructures.byte_flow import ByteFlow import dis from typing import Dict @@ -27,7 +29,6 @@ def __init__(self, byte_flow: ByteFlow): self.bcmap_from_bytecode(byte_flow.bc) self.rendered_blocks = set() - self.rendered_regions = set() self.render_region(self.g, None) self.render_edges() @@ -82,43 +83,33 @@ def render_region(self, graph, region_name): # If region name is none, we're in the 'root' region # that is the graph itself. if region_name is None: - region_headers = self.scfg.region_headers - all_blocks_iterator = self.scfg + region_name = self.scfg.meta_region + region = self.scfg.regions[region_name] else: region = self.scfg.regions[region_name] - region_headers = region.sub_region_headers - all_blocks_iterator = self.scfg.iterate_region(region_name) - - # If subregions exist within this region we render them first - if region_headers: - for _, regions in region_headers.items(): - for _region_name in regions: - _region = self.scfg.regions[_region_name] - with graph.subgraph(name=f"cluster_{_region_name}") as subg: - color = "blue" - if _region.kind == "branch": - color = "green" - if _region.kind == "tail": - color = "purple" - if _region.kind == "head": - color = "red" - subg.attr(color=color, label=_region.kind) - self.render_region(subg, _region_name) - - # If there are no further subregions then we render the blocks - for block_name in all_blocks_iterator: - self.render_block(graph, block_name, self.scfg[block_name]) - - def render_block(self, graph, block_name, block): + + all_blocks = self.scfg.iterate_region(region_name) + + with graph.subgraph(name=f"cluster_{region_name}") as subg: + if isinstance(region, LoopRegion): + color = "blue" + else: + color = "black" + subg.attr(color=color, label=str(region.label)) + + # If there are no further subregions then we render the blocks + for block_name in all_blocks: + self.render_block(subg, block_name) + + def render_block(self, graph, block_name): if block_name in self.rendered_blocks: return - if block_name in self.scfg.region_headers.keys(): - for region in self.scfg.region_headers[block_name]: - if region not in self.rendered_regions: - self.rendered_regions.add(region) - self.render_region(graph, region) + if isinstance(block_name, RegionName): + self.render_region(graph, block_name) + return + block = self.scfg[block_name] if type(block) == BasicBlock: self.render_basic_block(graph, block_name) elif type(block) == PythonBytecodeBlock: @@ -132,20 +123,20 @@ def render_block(self, graph, block_name, block): self.rendered_blocks.add(block_name) def render_edges(self): - for block_name, out_edges in self.scfg.out_edges.items(): for out_edge in out_edges: - self.g.edge(str(block_name), str(out_edge)) - - for block_name, back_edges in self.scfg.back_edges.items(): - for back_edge in back_edges: - self.g.edge( - str(block_name), - str(back_edge), - style="dashed", - color="grey", - constraint="0", - ) + if isinstance(out_edge, RegionName): + out_edge = self.scfg.regions[out_edge].header + if (block_name, out_edge) in self.scfg.back_edges: + self.g.edge( + str(block_name), + str(out_edge), + style="dashed", + color="grey", + constraint="0", + ) + else: + self.g.edge(str(block_name), str(out_edge)) def bcmap_from_bytecode(self, bc: dis.Bytecode): self.bcmap: Dict[int, dis.Instruction] = ByteFlow.bcmap_from_bytecode(bc) diff --git a/numba_rvsdg/tests/test_utils.py b/numba_rvsdg/tests/test_utils.py index 7020156..dc73e2f 100644 --- a/numba_rvsdg/tests/test_utils.py +++ b/numba_rvsdg/tests/test_utils.py @@ -18,7 +18,7 @@ def assertSCFGEqual(self, first_scfg: SCFG, second_scfg: SCFG): self.assertEqual(type(block_1.label), type(block_2.label)) # compare edges self.assertEqual(first_scfg.out_edges[key1], second_scfg.out_edges[key2]) - self.assertEqual(first_scfg.back_edges[key1], second_scfg.back_edges[key2]) + self.assertEqual(first_scfg.back_edges, second_scfg.back_edges) def assertYAMLEquals(self, first_yaml: str, second_yaml: str, ref_dict: Dict): for key, value in ref_dict.items(): From 7db364b8058beda55cc8051d632a10bacc197044 Mon Sep 17 00:00:00 2001 From: kc611 Date: Fri, 31 Mar 2023 19:39:58 +0530 Subject: [PATCH 24/30] Fixed Rendering --- numba_rvsdg/core/datastructures/scfg.py | 8 ++++---- numba_rvsdg/rendering/rendering.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index e7add0d..158fdf8 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -394,9 +394,6 @@ def to_dict(self): return graph_dict def iterate_region(self, region_name, region_view=False): - if region_name == self.meta_region and not region_view: - return iter(self) - region = self.regions[region_name] """Region Iterator""" region_head = region.header if region_name is not self.meta_region else self.find_head() @@ -416,7 +413,10 @@ def iterate_region(self, region_name, region_view=False): if region_name is not self.meta_region and block_name is region.exiting: continue # finally add any out_edges to the list of block_names to visit - outs = self.out_edges[block_name] + if isinstance(block_name, RegionName): + outs = self.out_edges[self.regions[block_name].exiting] + else: + outs = self.out_edges[block_name] if not region_view: for idx, _out in enumerate(outs): diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 2ef5cb1..341eb09 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -88,7 +88,7 @@ def render_region(self, graph, region_name): else: region = self.scfg.regions[region_name] - all_blocks = self.scfg.iterate_region(region_name) + all_blocks = list(self.scfg.iterate_region(region_name, region_view=True)) with graph.subgraph(name=f"cluster_{region_name}") as subg: if isinstance(region, LoopRegion): From cc0e95a49d93225b3b103f918d5def3285a42835 Mon Sep 17 00:00:00 2001 From: kc611 Date: Mon, 3 Apr 2023 15:58:02 +0530 Subject: [PATCH 25/30] Added tests for loops --- numba_rvsdg/core/datastructures/scfg.py | 5 +- numba_rvsdg/rendering/rendering.py | 18 +- numba_rvsdg/tests/test_transforms.py | 231 ++++++++++++++++++++++-- 3 files changed, 230 insertions(+), 24 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 158fdf8..4e3000d 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -180,7 +180,7 @@ def find_headers_and_entries( # the CFG. if not headers: headers.add(self.find_head()) - return sorted(list(headers)), sorted(list(entries)) + return sorted(headers), sorted(entries) def find_exiting_and_exits( self, subgraph: Set[BlockName] @@ -311,10 +311,9 @@ def add_region(self, kind: str, header: BlockName, exiting: BlockName, parent: R return region_name - def add_connections(self, block_name, out_edges=[]): + def add_connections(self, block_name, out_edges): assert self.out_edges[block_name] == [] self.out_edges[block_name] = out_edges - self.check_graph() diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 341eb09..a19643b 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -150,13 +150,17 @@ def view(self, *args): def render_func(func): flow = ByteFlow.from_bytecode(func) - ByteFlowRenderer(flow).view("before") + render_flow(flow) - flow._join_returns() - ByteFlowRenderer(flow).view("closed") - flow._restructure_loop() - ByteFlowRenderer(flow).view("loop restructured") +def render_flow(byte_flow): + ByteFlowRenderer(byte_flow).view("before") - flow._restructure_branch() - ByteFlowRenderer(flow).view("branch restructured") + byte_flow._join_returns() + ByteFlowRenderer(byte_flow).view("closed") + + byte_flow._restructure_loop() + ByteFlowRenderer(byte_flow).view("loop restructured") + + byte_flow._restructure_branch() + ByteFlowRenderer(byte_flow).view("branch restructured") diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index 67441e7..e42b998 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -43,7 +43,7 @@ def test_two_returns(self): label_type: "synth_return" out: [] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) join_returns(original_scfg) @@ -69,7 +69,7 @@ def test_join_tails_and_exits_case_00(self): type: "basic" out: [] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) tails = list((block_ref_orig["0"],)) exits = list((block_ref_orig["1"],)) @@ -111,7 +111,7 @@ def test_join_tails_and_exits_case_01(self): label_type: "synth_exit" out: ["1", "2"] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) tails = list((block_ref_orig["0"],)) exits = list((block_ref_orig["1"], block_ref_orig["2"])) @@ -153,7 +153,7 @@ def test_join_tails_and_exits_case_02_01(self): label_type: "synth_tail" out: ["3"] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) tails = list((block_ref_orig["1"], block_ref_orig["2"])) exits = list((block_ref_orig["3"],)) @@ -195,7 +195,7 @@ def test_join_tails_and_exits_case_02_02(self): label_type: "synth_tail" out: ["3"] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) tails = list((block_ref_orig["1"], block_ref_orig["2"])) exits = list((block_ref_orig["3"],)) @@ -254,7 +254,7 @@ def test_join_tails_and_exits_case_03_01(self): label_type: "synth_exit" out: ["3", "4"] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) tails = list((block_ref_orig["1"], block_ref_orig["2"])) exits = list((block_ref_orig["3"], block_ref_orig["4"])) @@ -313,7 +313,7 @@ def test_join_tails_and_exits_case_03_02(self): label_type: "synth_exit" out: ["3", "4"] """ - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) tails = list((block_ref_orig["1"], block_ref_orig["2"])) exits = list((block_ref_orig["3"], block_ref_orig["4"])) @@ -349,7 +349,7 @@ def test_no_op_mono(self): out: [] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) loop_restructure_helper(original_scfg, set((block_ref_orig["1"],))) @@ -387,7 +387,7 @@ def test_no_op(self): out: [] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"]))) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -438,7 +438,7 @@ def test_backedge_not_exiting(self): out: ["4"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"]))) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -500,7 +500,7 @@ def test_double_exit(self): out: ["5"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"]))) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -577,7 +577,7 @@ def test_double_header(self): out: ["9"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"], block_ref_orig["4"]))) self.assertSCFGEqual(expected_scfg, original_scfg) @@ -678,10 +678,213 @@ def test_double_header_double_exiting(self): out: ["12"] """ original_scfg, block_ref_orig = SCFG.from_yaml(original) - expected_scfg, block_ref_exp = SCFG.from_yaml(expected) + expected_scfg, _ = SCFG.from_yaml(expected) loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"], block_ref_orig["4"]))) self.assertSCFGEqual(expected_scfg, original_scfg) +class TestLoops(SCFGComparator): + def test_basic_for_loop(self): + + original = """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: ["2", "3"] + "2": + type: "basic" + out: ["1"] + "3": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: ["2", "5"] + "2": + type: "basic" + out: ["6"] + "3": + type: "basic" + out: [] + "4": + type: "basic" + label_type: "synth_exit_latch" + out: ["1", "3"] + back: ["1"] + "5": + type: "basic" + label_type: "synth_assign" + out: ["4"] + "6": + type: "basic" + label_type: "synth_assign" + out: ["4"] + """ + expected_scfg, _ = SCFG.from_yaml(expected) + + loop_restructure_helper(original_scfg, set((block_ref_orig["1"], block_ref_orig["2"]))) + print(original_scfg.compute_scc()) + self.assertSCFGEqual(expected_scfg, original_scfg) + + def test_basic_while_loop(self): + original = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["1", "2"] + "2": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + expected = """ + "0": + type: "basic" + out: ["1", "2"] + "1": + type: "basic" + out: ["1", "2"] + back: ["1"] + "2": + type: "basic" + out: [] + """ + expected_scfg, _ = SCFG.from_yaml(expected) + + loop_restructure_helper(original_scfg, set((block_ref_orig["1"],))) + print(original_scfg.compute_scc()) + self.assertSCFGEqual(expected_scfg, original_scfg) + + def test_mixed_for_while_loop_with_branch(self): + original = """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: ["2", "7"] + "2": + type: "basic" + out: ["3", "6"] + "3": + type: "basic" + out: ["4", "5"] + "4": + type: "basic" + out: ["5"] + "5": + type: "basic" + out: ["3", "6"] + "6": + type: "basic" + out: ["1"] + "7": + type: "basic" + out: [] + """ + original_scfg, block_ref_orig = SCFG.from_yaml(original) + # this has two loops, so we need to attempt to rotate twice, first for + # the header controlled loop, inserting an additional block + expected01 = """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: ["2", "9"] + "2": + type: "basic" + out: ["3", "6"] + "3": + type: "basic" + out: ["4", "5"] + "4": + type: "basic" + out: ["5"] + "5": + type: "basic" + out: ["3", "6"] + "6": + type: "basic" + out: ["10"] + "7": + type: "basic" + out: [] + "8": + type: "basic" + label_type: "synth_exit_latch" + out: ["1", "7"] + back: ["1"] + "9": + type: "basic" + label_type: "synth_assign" + out: ["8"] + "10": + type: "basic" + label_type: "synth_assign" + out: ["8"] + """ + expected01_block_map, _ = SCFG.from_yaml(expected01) + loop_restructure_helper(original_scfg, + set((block_ref_orig["1"], block_ref_orig["2"], block_ref_orig["3"], block_ref_orig["4"], block_ref_orig["5"], block_ref_orig["6"]))) + self.assertSCFGEqual(expected01_block_map, original_scfg) + # And then, we make sure that the inner-loop remains unchanged, and the + # loop rotation will only detect the aditional backedge, from 5 to 3 + expected02 = """ + "0": + type: "basic" + out: ["1"] + "1": + type: "basic" + out: ["2", "9"] + "2": + type: "basic" + out: ["3", "6"] + "3": + type: "basic" + out: ["4", "5"] + "4": + type: "basic" + out: ["5"] + "5": + type: "basic" + out: ["3", "6"] + back: ["3"] + "6": + type: "basic" + out: ["10"] + "7": + type: "basic" + out: [] + "8": + type: "basic" + label_type: "synth_exit_latch" + out: ["1", "7"] + back: ["1"] + "9": + type: "basic" + label_type: "synth_assign" + out: ["8"] + "10": + type: "basic" + label_type: "synth_assign" + out: ["8"] + """ + expected02_block_map, _ = SCFG.from_yaml(expected02) + loop_restructure_helper(original_scfg, + set((block_ref_orig["3"], block_ref_orig["4"], block_ref_orig["5"],))) + self.assertSCFGEqual(expected02_block_map, original_scfg) + + if __name__ == "__main__": - main() + main() \ No newline at end of file From a86040fae08707e796a3e53202e552f9cae57874 Mon Sep 17 00:00:00 2001 From: kc611 Date: Mon, 3 Apr 2023 16:41:02 +0530 Subject: [PATCH 26/30] Fixed review comments --- numba_rvsdg/core/datastructures/scfg.py | 10 +++++----- numba_rvsdg/tests/test_transforms.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index 4e3000d..a60abbe 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -30,7 +30,7 @@ class SCFG: regions: Dict[RegionName, Region] = field(default_factory=dict, init=False) meta_region: RegionName = field(init=False) - region_tree: Dict[BlockName, List[RegionName]] = field(default_factory=dict, init=False) + region_tree: Dict[RegionName, List[RegionName]] = field(default_factory=dict, init=False) name_gen: NameGenerator = field(default_factory=NameGenerator, compare=False, init=False) @@ -50,7 +50,7 @@ def __contains__(self, index: BlockName) -> bool: def __iter__(self): """Graph Iterator""" # initialise housekeeping datastructures - to_visit, seen = [self.find_head()], [] + to_visit, seen = [self.find_head()], set() while to_visit: # get the next name on the list name = to_visit.pop(0) @@ -58,7 +58,7 @@ def __iter__(self): if name in seen: continue else: - seen.append(name) + seen.add(name) # get the corresponding block for the name if isinstance(name, RegionName): name = self.regions[name].header @@ -398,7 +398,7 @@ def iterate_region(self, region_name, region_view=False): region_head = region.header if region_name is not self.meta_region else self.find_head() # initialise housekeeping datastructures - to_visit, seen = [region_head], [] + to_visit, seen = [region_head], set while to_visit: # get the next block_name on the list block_name = to_visit.pop(0) @@ -406,7 +406,7 @@ def iterate_region(self, region_name, region_view=False): if block_name in seen: continue else: - seen.append(block_name) + seen.add(block_name) # yield the block_name yield block_name if region_name is not self.meta_region and block_name is region.exiting: diff --git a/numba_rvsdg/tests/test_transforms.py b/numba_rvsdg/tests/test_transforms.py index e42b998..859de1b 100644 --- a/numba_rvsdg/tests/test_transforms.py +++ b/numba_rvsdg/tests/test_transforms.py @@ -887,4 +887,4 @@ def test_mixed_for_while_loop_with_branch(self): if __name__ == "__main__": - main() \ No newline at end of file + main() From 66729dfe59791a9e807dd52d877c3bffc9b2c483 Mon Sep 17 00:00:00 2001 From: esc Date: Tue, 4 Apr 2023 09:54:53 +0200 Subject: [PATCH 27/30] fix set intialization in region iterator As title --- numba_rvsdg/core/datastructures/scfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index a60abbe..bd27dc2 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -398,7 +398,7 @@ def iterate_region(self, region_name, region_view=False): region_head = region.header if region_name is not self.meta_region else self.find_head() # initialise housekeeping datastructures - to_visit, seen = [region_head], set + to_visit, seen = [region_head], set() while to_visit: # get the next block_name on the list block_name = to_visit.pop(0) From dd17688814096fbd02dc0af5598799a655a318f1 Mon Sep 17 00:00:00 2001 From: esc Date: Tue, 4 Apr 2023 09:56:08 +0200 Subject: [PATCH 28/30] fix conversion of out_targets in implicit jumps As title --- numba_rvsdg/core/datastructures/flow_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numba_rvsdg/core/datastructures/flow_info.py b/numba_rvsdg/core/datastructures/flow_info.py index f0e8a18..dbc3cb9 100644 --- a/numba_rvsdg/core/datastructures/flow_info.py +++ b/numba_rvsdg/core/datastructures/flow_info.py @@ -89,7 +89,7 @@ def build_basicblocks(self: "FlowInfo", end_offset=None) -> "SCFG": term_offset = _prev_inst_offset(end) if term_offset not in self.jump_insts: # implicit jump - targets = (names[end],) + targets = [names[end],] else: targets = [names[o] for o in self.jump_insts[term_offset]] From c750c2ac196038ff0331a8383830bf495e7fe939 Mon Sep 17 00:00:00 2001 From: esc Date: Tue, 4 Apr 2023 16:30:44 +0200 Subject: [PATCH 29/30] WIP As title --- numba_rvsdg/core/datastructures/byte_flow.py | 3 +- numba_rvsdg/core/datastructures/scfg.py | 97 +++++++++++++++++++- numba_rvsdg/core/transformations.py | 28 +++++- numba_rvsdg/rendering/rendering.py | 5 +- 4 files changed, 120 insertions(+), 13 deletions(-) diff --git a/numba_rvsdg/core/datastructures/byte_flow.py b/numba_rvsdg/core/datastructures/byte_flow.py index 8274abe..0fe5ea6 100644 --- a/numba_rvsdg/core/datastructures/byte_flow.py +++ b/numba_rvsdg/core/datastructures/byte_flow.py @@ -7,6 +7,7 @@ from numba_rvsdg.core.transformations import ( restructure_loop, + restructure_loop_recursive, restructure_branch, join_returns, ) @@ -30,7 +31,7 @@ def _join_returns(self): join_returns(self.scfg) def _restructure_loop(self): - restructure_loop(self.scfg) + restructure_loop_recursive(self.scfg) def _restructure_branch(self): restructure_branch(self.scfg) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index bd27dc2..8848754 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -4,10 +4,12 @@ from textwrap import dedent from typing import Set, Tuple, Dict, List, Iterator from dataclasses import dataclass, field +from collections import deque from numba_rvsdg.core.datastructures.basic_block import BasicBlock, get_block_class, get_block_class_str from numba_rvsdg.core.datastructures.region import MetaRegion, Region, LoopRegion, get_region_class from numba_rvsdg.core.datastructures.labels import ( + Name, Label, BlockName, NameGenerator, @@ -118,7 +120,7 @@ def compute_scc_subgraph(self, subgraph) -> List[Set[BlockName]]: """ from numba_rvsdg.networkx_vendored.scc import scc - out_edges = self.out_edges + scfg = self class GraphWrap: def __init__(self, graph: Dict[BlockName, BasicBlock], subgraph): @@ -126,9 +128,11 @@ def __init__(self, graph: Dict[BlockName, BasicBlock], subgraph): self.subgraph = subgraph def __getitem__(self, vertex): - out = out_edges[vertex] + + out = scfg.get_out_edges(vertex, region_view=False) # Exclude node outside of the subgraph - return [k for k in out if k in subgraph] + return [k for k in out if k in subgraph + and not (vertex, k) in scfg.back_edges] def __iter__(self): return iter(self.graph.keys()) @@ -171,7 +175,8 @@ def find_headers_and_entries( for outside in self.exclude_blocks(subgraph): # Check if the current block points to any blocks that are inside # the subgraph. - targets_in_loop = subgraph.intersection(self.out_edges[outside]) + targets_in_loop = subgraph.intersection(self.get_out_edges(outside, + region_view=False)) # Record both headers and entries if targets_in_loop: headers.update(targets_in_loop) @@ -392,6 +397,88 @@ def to_dict(self): return graph_dict + def blocks_in_region(self, region_name: RegionName): + """Generator for all blocks in a given region. + + Parameters + ---------- + region_name: RegionName + + Returns + ------- + gen: generator of BlockName + A generator that yields block names + """ + # get correct region from, scfg + region = self.regions[region_name] + # initialize housekeeping datastructures + to_visit, seen = [region.header], set() + while to_visit: + # get the next block_name on the list + name = to_visit.pop(0) + # if we have visited this, we skip it + if name in seen: + continue + # otherwise add it to the list of seen names + else: + seen.add(name) + # unless this is the exiting block of the region + if name is not region.exiting: + out_edges = [e for e in + self.get_out_edges(name, region_view=False) + if not isinstance(e, RegionName) + ] + to_visit.extend(out_edges) + yield name + + def get_out_edges(self, name: Name, region_view: bool = True): + if region_view: + if isinstance(name, RegionName): + if name is self.meta_region: + raise ValueError("Meta Region encompasses all the graph, it cannot have out edges") + name = self.regions[name].exiting + else: + if isinstance(name, RegionName): + if name is self.meta_region: + raise ValueError("Meta Region encompasses all the graph, it cannot have out edges") + name = self.regions[name].header + + return self.out_edges[name] + + def region_iterator(self) -> Iterator[RegionName]: + """A region iterator that is aware of new regions being added. + + This iterator returns `RegionName` from the `regions` dictionary. + Importantly, this iterator has an internal queue that is updated with + new keys from the `regions` dictionary on every iteration. This allows + the `regions` dictionary to be updated while this iterator is running. + + Returns + ------- + iterator: Iterator[ReionName] + the regions of this scfg + + """ + # initialize housekeeping datastructures + queue, seen = deque(), set() + while True: + # extend the queue with any un-seen items + queue.extend([r for r in self.regions if r not in seen]) + # If the queue is now empty exit + if not queue: + break + # Otherwise... + else: + # get the next region name + r = queue.popleft() + if r in seen: + continue + # add it to the set of seen regions + seen.add(r) + # finally, yield the region name + yield r + + def iterate_region(self, region_name, region_view=False): region = self.regions[region_name] """Region Iterator""" @@ -420,7 +507,7 @@ def iterate_region(self, region_name, region_view=False): if not region_view: for idx, _out in enumerate(outs): if isinstance(_out, RegionName): - to_visit.append(_out.header) + to_visit.append(self.regions[_out].header) else: to_visit.append(_out) else: diff --git a/numba_rvsdg/core/transformations.py b/numba_rvsdg/core/transformations.py index e78022c..bb28e8b 100644 --- a/numba_rvsdg/core/transformations.py +++ b/numba_rvsdg/core/transformations.py @@ -216,13 +216,16 @@ def reverse_lookup(d, value): scfg.out_edges[synth_exiting_latch].append(exit_blocks[0]) -def restructure_loop(scfg: SCFG): +def restructure_loop(scfg: SCFG, subgraph: list[BlockName] = None): """Inplace restructuring of the given graph to extract loops using strongly-connected components """ # obtain a List of Sets of Labels, where all labels in each set are strongly # connected, i.e. all reachable from one another by traversing the subset - scc: List[Set[SCFG]] = scfg.compute_scc() + if subgraph: + scc: List[Set[SCFG]] = scfg.compute_scc_subgraph(subgraph) + else: + scc: List[Set[SCFG]] = scfg.compute_scc() # loops are defined as strongly connected subsets who have more than a # single label and single label loops that point back to to themselves. loops: List[Set[SCFG]] = [ @@ -235,9 +238,24 @@ def restructure_loop(scfg: SCFG): "restructure_loop found %d loops in %s", len(loops), scfg.blocks.keys() ) # rotate and extract loop - for loop in loops: - loop_restructure_helper(scfg, loop) - extract_region(scfg, loop, "loop") + for l in loops: + loop_restructure_helper(scfg, l) + extract_region(scfg, l, "loop") + + +def restructure_loop_recursive(scfg: SCFG): + # find all top-level loops + restructure_loop(scfg) + # if any loops were found, continue down into regions + for region_name in scfg.region_iterator(): + if region_name is scfg.meta_region: + # skip the meta region, it doesn't have header or exiting + continue + region_blocks = list(scfg.blocks_in_region(region_name)) + #region_blocks = list(scfg.iterate_region(region_name)) + #breakpoint() + breakpoint() + restructure_loop(scfg, subgraph=region_blocks) def find_head_blocks(scfg: SCFG, begin: BlockName) -> Set[BlockName]: diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index a19643b..0f1aee7 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -160,7 +160,8 @@ def render_flow(byte_flow): ByteFlowRenderer(byte_flow).view("closed") byte_flow._restructure_loop() + breakpoint() ByteFlowRenderer(byte_flow).view("loop restructured") - byte_flow._restructure_branch() - ByteFlowRenderer(byte_flow).view("branch restructured") + #byte_flow._restructure_branch() + #ByteFlowRenderer(byte_flow).view("branch restructured") From 53f62064d52ad010fa4514f578a835efba5af71c Mon Sep 17 00:00:00 2001 From: esc Date: Tue, 4 Apr 2023 16:41:48 +0200 Subject: [PATCH 30/30] fix stackoverflow caused by recursion As title --- numba_rvsdg/rendering/rendering.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numba_rvsdg/rendering/rendering.py b/numba_rvsdg/rendering/rendering.py index 0f1aee7..e43dd13 100644 --- a/numba_rvsdg/rendering/rendering.py +++ b/numba_rvsdg/rendering/rendering.py @@ -106,6 +106,7 @@ def render_block(self, graph, block_name): return if isinstance(block_name, RegionName): + self.rendered_blocks.add(block_name) self.render_region(graph, block_name) return