Fix sqd preprocessing, namespace package checks (#1203)

scripts/demos.py

@@ -17,6 +17,7 @@
     **dict(os.environ),
     'POSTGRES_PASSWORD': '',
     'HASURA_SECRET': '',
+    'DIPDUP_NO_SYMLINK': '1',
 }
 
 
@@ -49,9 +50,6 @@ def _init_demo(path: Path) -> None:
         env=DEFAULT_ENV,
     )
 
-    # NOTE: We don't need magic symlinks in demo projects.
-    Path(package_path).joinpath(package).unlink(missing_ok=True)
-
 
 def _rm_demo(path: Path) -> None:
     rmtree(path, ignore_errors=True)

src/demo_blank/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_blank"
 

src/demo_evm_events/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_evm_events"
 

src/demo_evm_transactions/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_evm_transactions"
 

src/demo_evm_uniswap/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_evm_uniswap"
 

src/demo_starknet_events/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_starknet_events"
 

src/demo_substrate_events/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_substrate_events"
 

src/demo_tezos_auction/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_auction"
 

src/demo_tezos_dao/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_dao"
 

src/demo_tezos_dex/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_dex"
 

src/demo_tezos_domains/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_domains"
 

src/demo_tezos_events/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_events"
 

src/demo_tezos_factories/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_factories"
 

src/demo_tezos_head/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_head"
 

src/demo_tezos_nft_marketplace/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_nft_marketplace"
 

src/demo_tezos_raw/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_raw"
 

src/demo_tezos_token/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_token"
 

src/demo_tezos_token_balances/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_token_balances"
 

src/demo_tezos_token_transfers/pyproject.toml

@@ -38,6 +38,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "demo_tezos_token_transfers"
 

src/dipdup/package.py

@@ -63,8 +63,8 @@ def __init__(self, root: Path, quiet: bool = False) -> None:
         self.name = root.name
 
         # NOTE: Paths expected to exist in package root
-        self.pyproject = root.parent / 'pyproject.toml'
-        self.root_config = root.parent / 'dipdup.yaml'
+        self.pyproject = root / 'pyproject.toml'
+        self.root_config = root / 'dipdup.yaml'
 
         # NOTE: Package sections with .keep markers
         self.abi = root / 'abi'
@@ -130,7 +130,6 @@ def skel(self) -> dict[Path, str | None]:
             self.types: '**/*.py',
             # NOTE: Python metadata
             Path(PEP_561_MARKER): None,
-            Path(PACKAGE_MARKER): None,
         }
 
     def in_migration(self) -> bool:
@@ -180,7 +179,11 @@ def act(x: str) -> None:
 
     def _post_init(self) -> None:
         # NOTE: Allows plain package structure to be imported
-        if self.root != Path.cwd() or env.NO_SYMLINK:
+        if env.NO_SYMLINK:
+            touch(self.root / PACKAGE_MARKER)
+            return
+
+        if self.root != Path.cwd():
             return
 
         symlink_path = self.root.joinpath(self.name)

src/dipdup/projects/base/pyproject.toml.j2

@@ -58,6 +58,7 @@ isort = { force-single-line = true}
 [tool.mypy]
 python_version = "3.12"
 plugins = ["pydantic.mypy"]
+explicit_package_bases = true
 strict = false
 exclude = "{{ project.package }}"
 

src/dipdup/runtimes.py

@@ -281,11 +281,8 @@ def extract_subsquid_payload(data: Any) -> Any:
         if 'value' in data:
             value = data['value']
             if isinstance(value, list | tuple):
-                # Handle list of values
                 value = tuple(extract_subsquid_payload(item) for item in value)
-            # FIXME: We probably shouldn't do this
-            elif isinstance(value, str):
-                value = int(value)
+
             return {kind: value}
 
         # NOTE: Special case

tests/test_datasources/test_substrate.py

@@ -65,7 +65,7 @@
             'interior': {
                 'X2': (
                     {'PalletInstance': 50},
-                    {'GeneralIndex': 1337},
+                    {'GeneralIndex': '1337'},
                 ),
             },
         },