refactor: support dynamic algo module (#103)

* refactor: support dynamic algo module

* chore: format code

dbterd/adapters/algos/base.py

@@ -490,6 +490,54 @@ def get_relationships(manifest: Manifest, **kwargs) -> List[Ref]:
     return get_unique_refs(refs=refs)
 
 
+# def get_relationships_by_contraints(manifest: Manifest, **kwargs) -> List[Ref]:
+#     """Extract relationships from dbt artifacts based on model's configured contraints
+
+#     Args:
+#         manifest (dict): Manifest json
+
+#     Returns:
+#         List[Ref]: List of parsed relationship
+#     """
+#     wfk_nodes = [
+#         (node, constraint)
+#         for node in manifest.nodes
+#         if not node.startswith("test") and node.constraints is not None
+#         for constraint in node.constraints
+#         if constraint.type == "foreign_key"
+#     ]
+
+#     refs = []
+#     for x in wfk_nodes:
+#         to_model_expr = str(x[1].expression).split("(")
+#         to_model_columns_str = to_model_expr[1].strip()[:-1]
+#         to_model_name = to_model_expr[0].split(".")[-1].strip()
+#         find_to_models = [
+#             str(node)
+#             for node in manifest.nodes
+#             if str(node).endswith(f".{to_model_name}")
+#         ]
+#         if find_to_models:
+#             to_model_name = find_to_models[0]
+
+#         refs.append(
+#             Ref(
+#                 name=x[1].name,
+#                 table_map=[
+#                     to_model_name,
+#                     str(x[0]),
+#                 ],
+#                 column_map=[
+#                     to_model_columns_str.replace('"', "").lower(),
+#                     str(",".join(x[1].columns)).replace('"', "").lower(),
+#                 ],
+#                 type="1n",  # cannot add `relationship_type` meta to constraints
+#             )
+#         )
+
+#     return get_unique_refs(refs=refs)
+
+
 def make_up_relationships(
     relationships: List[Ref] = [], tables: List[Table] = []
 ) -> List[Ref]:

dbterd/adapters/algos/test_relationship.py

@@ -46,7 +46,10 @@ def parse_metadata(data, **kwargs) -> Tuple[List[Table], List[Ref]]:
     logger.info(
         f"Collected {len(tables)} table(s) and {len(relationships)} relationship(s)"
     )
-    return (tables, relationships)
+    return (
+        sorted(tables, key=lambda tbl: tbl.node_name),
+        sorted(relationships, key=lambda rel: rel.name),
+    )
 
 
 def parse(

dbterd/adapters/base.py

@@ -173,14 +173,7 @@ def __get_operation(self, kwargs):
             func: Operation function
         """
         target = adapter.load_target(name=kwargs["target"])  # import {target}
-        run_operation_dispatcher = getattr(target, "run_operation_dispatcher")
-        operation_default = getattr(target, "run_operation_default")
-        operation = run_operation_dispatcher.get(
-            f"{kwargs['target']}_{kwargs['algo'].split(':')[0]}",
-            operation_default,
-        )
-
-        return operation
+        return getattr(target, "run")
 
     def __save_result(self, path, data):
         """Save ERD data to file

dbterd/adapters/targets/d2/d2_test_relationship.py → dbterd/adapters/targets/d2.py

@@ -1,6 +1,6 @@
 from typing import Tuple
 
-from dbterd.adapters.algos import test_relationship
+from dbterd.adapters import adapter
 from dbterd.types import Catalog, Manifest
 
 
@@ -28,7 +28,8 @@ def parse(manifest: Manifest, catalog: Catalog, **kwargs) -> str:
     Returns:
         str: D2 content
     """
-    tables, relationships = test_relationship.parse(
+    algo_module = adapter.load_algo(name=kwargs["algo"])
+    tables, relationships = algo_module.parse(
         manifest=manifest, catalog=catalog, **kwargs
     )
 

dbterd/adapters/targets/dbml/dbml_test_relationship.py → dbterd/adapters/targets/dbml.py

@@ -1,7 +1,7 @@
 import json
 from typing import Tuple
 
-from dbterd.adapters.algos import test_relationship
+from dbterd.adapters import adapter
 from dbterd.types import Catalog, Manifest
 
 
@@ -29,7 +29,9 @@ def parse(manifest: Manifest, catalog: Catalog, **kwargs) -> str:
     Returns:
         str: DBML content
     """
-    tables, relationships = test_relationship.parse(
+
+    algo_module = adapter.load_algo(name=kwargs["algo"])
+    tables, relationships = algo_module.parse(
         manifest=manifest, catalog=catalog, **kwargs
     )
 

dbterd/adapters/targets/graphviz/graphviz_test_relationship.py → dbterd/adapters/targets/graphviz.py

@@ -1,6 +1,6 @@
 from typing import Tuple
 
-from dbterd.adapters.algos import test_relationship
+from dbterd.adapters import adapter
 from dbterd.types import Catalog, Manifest
 
 
@@ -28,7 +28,8 @@ def parse(manifest: Manifest, catalog: Catalog, **kwargs) -> str:
     Returns:
         str: GraphViz content
     """
-    tables, relationships = test_relationship.parse(
+    algo_module = adapter.load_algo(name=kwargs["algo"])
+    tables, relationships = algo_module.parse(
         manifest=manifest, catalog=catalog, **kwargs
     )
 

dbterd/adapters/targets/mermaid/mermaid_test_relationship.py → dbterd/adapters/targets/mermaid.py

@@ -1,7 +1,7 @@
 import re
 from typing import Optional, Tuple
 
-from dbterd.adapters.algos import test_relationship
+from dbterd.adapters import adapter
 from dbterd.types import Catalog, Manifest
 
 
@@ -79,7 +79,8 @@ def parse(manifest: Manifest, catalog: Catalog, **kwargs) -> str:
     Returns:
         str: Mermaid content
     """
-    tables, relationships = test_relationship.parse(
+    algo_module = adapter.load_algo(name=kwargs["algo"])
+    tables, relationships = algo_module.parse(
         manifest=manifest, catalog=catalog, **kwargs
     )
 

dbterd/adapters/targets/plantuml/plantuml_test_relationship.py → dbterd/adapters/targets/plantuml.py

@@ -1,6 +1,6 @@
 from typing import Tuple
 
-from dbterd.adapters.algos import test_relationship
+from dbterd.adapters import adapter
 from dbterd.types import Catalog, Manifest
 
 
@@ -28,7 +28,8 @@ def parse(manifest: Manifest, catalog: Catalog, **kwargs) -> str:
     Returns:
         str: PlantUML content
     """
-    tables, relationships = test_relationship.parse(
+    algo_module = adapter.load_algo(name=kwargs["algo"])
+    tables, relationships = algo_module.parse(
         manifest=manifest, catalog=catalog, **kwargs
     )
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dbterd"
-version = "1.0.0"
+version = "0.0.0"
 description = "Generate the ERD-as-a-code from dbt artifacts"
 authors = ["Dat Nguyen <[email protected]>"]
 license = "MIT"

tests/unit/adapters/targets/d2/test_d2_test_relationship.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbterd.adapters.meta import Column, Ref, Table
-from dbterd.adapters.targets.d2 import d2_test_relationship as engine
+from dbterd.adapters.targets import d2 as engine
 
 
 class TestD2TestRelationship:
@@ -248,6 +248,7 @@ def test_parse(
                     select=select,
                     exclude=exclude,
                     resource_type=resource_type,
+                    algo="test_relationship",
                 )
                 print("mermaid ", mermaid.replace(" ", "").replace("\n", ""))
                 print("expected", expected.replace(" ", "").replace("\n", ""))

tests/unit/adapters/targets/dbml/test_dbml_test_relationship.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbterd.adapters.meta import Column, Ref, Table
-from dbterd.adapters.targets.dbml import dbml_test_relationship as engine
+from dbterd.adapters.targets import dbml as engine
 
 
 class TestDbmlTestRelationship:
@@ -343,6 +343,7 @@ def test_parse(
                     select=select,
                     exclude=exclude,
                     resource_type=resource_type,
+                    algo="test_relationship",
                 )
                 assert dbml.replace(" ", "").replace("\n", "") == str(expected).replace(
                     " ", ""

tests/unit/adapters/targets/graphviz/test_graphviz_test_relationship.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbterd.adapters.meta import Column, Ref, Table
-from dbterd.adapters.targets.graphviz import graphviz_test_relationship as engine
+from dbterd.adapters.targets import graphviz as engine
 
 
 class TestGraphVizTestRelationship:
@@ -364,6 +364,7 @@ def test_parse(
                     select=select,
                     exclude=exclude,
                     resource_type=resource_type,
+                    algo="test_relationship",
                 )
                 print("graphviz ", graphviz.replace(" ", "").replace("\n", ""))
                 print("expected", expected.replace(" ", "").replace("\n", ""))

tests/unit/adapters/targets/mermaid/test_mermaid_test_relationship.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbterd.adapters.meta import Column, Ref, Table
-from dbterd.adapters.targets.mermaid import mermaid_test_relationship as engine
+from dbterd.adapters.targets import mermaid as engine
 
 
 class TestMermaidTestRelationship:
@@ -325,6 +325,7 @@ def test_parse(
                     exclude=exclude,
                     omit_columns=omit_columns,
                     resource_type=resource_type,
+                    algo="test_relationship",
                 )
                 print("mermaid ", mermaid.replace(" ", "").replace("\n", ""))
                 print("expected", expected.replace(" ", "").replace("\n", ""))

tests/unit/adapters/targets/plantuml/test_plantuml_test_relationship.py

@@ -3,7 +3,7 @@
 import pytest
 
 from dbterd.adapters.meta import Column, Ref, Table
-from dbterd.adapters.targets.plantuml import plantuml_test_relationship as engine
+from dbterd.adapters.targets import plantuml as engine
 
 
 class TestPlantUMLTestRelationship:
@@ -246,6 +246,7 @@ def test_parse(
                     select=select,
                     exclude=exclude,
                     resource_type=resource_type,
+                    algo="test_relationship",
                 )
                 print("plantuml ", plantuml.replace(" ", "").replace("\n", ""))
                 print("expected", expected.replace(" ", "").replace("\n", ""))

tests/unit/adapters/test_base.py

@@ -7,7 +7,7 @@
 from dbterd import default
 from dbterd.adapters.base import Executor
 from dbterd.adapters.dbt_core.dbt_invocation import DbtInvocation
-from dbterd.adapters.targets.dbml import dbml_test_relationship
+from dbterd.adapters.targets import dbml
 
 
 class TestBase:
@@ -30,14 +30,14 @@ def test___run_metadata_by_strategy(self, mock_query_erd_data, mock_save_result)
     @mock.patch("dbterd.adapters.base.DbtCloudMetadata.query_erd_data")
     @mock.patch("dbterd.adapters.base.Executor._Executor__save_result")
     def test___run_metadata_by_strategy_with_not_implemented_algo(
-        self, mock_query_erd_data, mock_save_result
+        self, mock_save_result, mock_query_erd_data
     ):
-        result = Executor(
-            ctx=click.Context(command=click.BaseCommand("dummy"))
-        )._Executor__run_metadata_by_strategy(target="dbml", algo="notfound")
-        assert result is None
+        with pytest.raises(Exception):
+            Executor(
+                ctx=click.Context(command=click.BaseCommand("dummy"))
+            )._Executor__run_metadata_by_strategy(target="dbml", algo="notfound")
         mock_query_erd_data.assert_called_once()
-        mock_save_result.assert_called_once()
+        mock_save_result.call_count == 0
 
     @mock.patch("builtins.open")
     def test___save_result(self, mock_open):
@@ -246,7 +246,7 @@ def test__set_single_node_selection(self, mock_find_related_nodes_by_id):
         ) == dict(algo="test_relationship", select=["irr"], exclude=[])
         assert mock_find_related_nodes_by_id.call_count == 1
 
-    @mock.patch("dbterd.adapters.targets.dbml.dbml_test_relationship.run")
+    @mock.patch("dbterd.adapters.targets.dbml.run")
     @mock.patch("dbterd.adapters.base.Executor._Executor__get_operation")
     @mock.patch("dbterd.adapters.base.Executor._Executor__read_manifest")
     @mock.patch("dbterd.adapters.base.Executor._Executor__read_catalog")
@@ -262,7 +262,7 @@ def test__run_by_strategy__for_api_simple(
         worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy")))
 
         mock_parent = mock.Mock()
-        mock_get_operation.return_value = dbml_test_relationship.run
+        mock_get_operation.return_value = dbml.run
         mock_parent.attach_mock(mock_get_operation, "mock_get_operation")
         mock_read_catalog.return_value = dict()
         mock_parent.attach_mock(mock_read_catalog, "mock_read_catalog")