diff --git a/CHANGELOG.md b/CHANGELOG.md index f6eae8cb7..66da2e5e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,16 @@ Releases prior to 7.0 has been removed from this file to declutter search result ## [Unreleased] +### Added + +- substrate.events: Added `subtrate.events` index kind to process Substrate events. +- substrate.node: Added `subtrate.node` datasource to receive data from Substrate node. +- substrate.subscan: Added `substrate.subscan` datasource to fetch ABIs from Subscan. +- substrate.subsquid: Added `substrate.subsquid` datasource to fetch historical data from Squid Network. + ### Fixed +- cli: Don't wrap exceptions with `CallbackError` to avoid shadowing the original exception. - cli: Fixed `--template` option being ignored when `--quiet` flag is set. - config: Fixed setting default loglevels when `logging` is a dict. diff --git a/docs/9.release-notes/_8.0_changelog.md b/docs/9.release-notes/_8.0_changelog.md index 23b539d25..db9537de1 100644 --- a/docs/9.release-notes/_8.0_changelog.md +++ b/docs/9.release-notes/_8.0_changelog.md @@ -18,12 +18,17 @@ - starknet.events: Added `starknet.events` index kind to process Starknet events. - starknet.node: Added Starknet node datasource for last mile indexing. - starknet.subsquid: Added `starknet.subsquid` datasource to fetch historical data from Subsquid Archives. +- substrate.events: Added `subtrate.events` index kind to process Substrate events. +- substrate.node: Added `subtrate.node` datasource to receive data from Substrate node. +- substrate.subscan: Added `substrate.subscan` datasource to fetch ABIs from Subscan. +- substrate.subsquid: Added `substrate.subsquid` datasource to fetch historical data from Squid Network. - tezos.operations: Added `sr_cement` operation type to process Smart Rollup Cemented Commitments. ### Fixed - cli: Don't save reports for successful test runs. - cli: Don't update existing installation in `self install` command unless asked to. +- cli: Don't wrap exceptions with `CallbackError` to avoid shadowing the original exception. - cli: Fixed `--pre` installer flag. - cli: Fixed `--template` option being ignored when `--quiet` flag is set. - cli: Fixed env files not being loaded in some commands. diff --git a/pdm.lock b/pdm.lock index 103361ef5..154441350 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "docs", "lint", "migrations", "perf", "test"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:0e1c65db1fbdd2e0588168a1a71c254498a2b55cac5dcefee53e6deb5a27f761" +content_hash = "sha256:dbc9da0fb29a2acc56dccbc29e05c5a44b0082beb4181402f23a2fb4faee73e0" [[metadata.targets]] requires_python = ">=3.12,<3.13" @@ -225,23 +225,23 @@ files = [ [[package]] name = "asyncpg" -version = "0.29.0" +version = "0.30.0" requires_python = ">=3.8.0" summary = "An asyncio PostgreSQL driver" groups = ["default"] dependencies = [ - "async-timeout>=4.0.3; python_version < \"3.12.0\"", + "async-timeout>=4.0.3; python_version < \"3.11.0\"", ] files = [ - {file = "asyncpg-0.29.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6011b0dc29886ab424dc042bf9eeb507670a3b40aece3439944006aafe023178"}, - {file = "asyncpg-0.29.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b544ffc66b039d5ec5a7454667f855f7fec08e0dfaf5a5490dfafbb7abbd2cfb"}, - {file = "asyncpg-0.29.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d84156d5fb530b06c493f9e7635aa18f518fa1d1395ef240d211cb563c4e2364"}, - {file = "asyncpg-0.29.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54858bc25b49d1114178d65a88e48ad50cb2b6f3e475caa0f0c092d5f527c106"}, - {file = "asyncpg-0.29.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bde17a1861cf10d5afce80a36fca736a86769ab3579532c03e45f83ba8a09c59"}, - {file = "asyncpg-0.29.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:37a2ec1b9ff88d8773d3eb6d3784dc7e3fee7756a5317b67f923172a4748a175"}, - {file = "asyncpg-0.29.0-cp312-cp312-win32.whl", hash = "sha256:bb1292d9fad43112a85e98ecdc2e051602bce97c199920586be83254d9dafc02"}, - {file = "asyncpg-0.29.0-cp312-cp312-win_amd64.whl", hash = "sha256:2245be8ec5047a605e0b454c894e54bf2ec787ac04b1cb7e0d3c67aa1e32f0fe"}, - {file = "asyncpg-0.29.0.tar.gz", hash = "sha256:d1c49e1f44fffafd9a55e1a9b101590859d881d639ea2922516f5d9c512d354e"}, + {file = "asyncpg-0.30.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c902a60b52e506d38d7e80e0dd5399f657220f24635fee368117b8b5fce1142e"}, + {file = "asyncpg-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aca1548e43bbb9f0f627a04666fedaca23db0a31a84136ad1f868cb15deb6e3a"}, + {file = "asyncpg-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c2a2ef565400234a633da0eafdce27e843836256d40705d83ab7ec42074efb3"}, + {file = "asyncpg-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1292b84ee06ac8a2ad8e51c7475aa309245874b61333d97411aab835c4a2f737"}, + {file = "asyncpg-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f5712350388d0cd0615caec629ad53c81e506b1abaaf8d14c93f54b35e3595a"}, + {file = "asyncpg-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:db9891e2d76e6f425746c5d2da01921e9a16b5a71a1c905b13f30e12a257c4af"}, + {file = "asyncpg-0.30.0-cp312-cp312-win32.whl", hash = "sha256:68d71a1be3d83d0570049cd1654a9bdfe506e794ecc98ad0873304a9f35e411e"}, + {file = "asyncpg-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a0292c6af5c500523949155ec17b7fe01a00ace33b68a476d6b5059f9630305"}, + {file = "asyncpg-0.30.0.tar.gz", hash = "sha256:c551e9928ab6707602f44811817f82ba3c446e018bfe1d3abecc8ba5f3eac851"}, ] [[package]] @@ -1111,7 +1111,7 @@ files = [ [[package]] name = "mypy" -version = "1.12.0" +version = "1.13.0" requires_python = ">=3.8" summary = "Optional static typing for Python" groups = ["lint"] @@ -1121,13 +1121,13 @@ dependencies = [ "typing-extensions>=4.6.0", ] files = [ - {file = "mypy-1.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8462655b6694feb1c99e433ea905d46c478041a8b8f0c33f1dab00ae881b2164"}, - {file = "mypy-1.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:923ea66d282d8af9e0f9c21ffc6653643abb95b658c3a8a32dca1eff09c06475"}, - {file = "mypy-1.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1ebf9e796521f99d61864ed89d1fb2926d9ab6a5fab421e457cd9c7e4dd65aa9"}, - {file = "mypy-1.12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e478601cc3e3fa9d6734d255a59c7a2e5c2934da4378f3dd1e3411ea8a248642"}, - {file = "mypy-1.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:c72861b7139a4f738344faa0e150834467521a3fba42dc98264e5aa9507dd601"}, - {file = "mypy-1.12.0-py3-none-any.whl", hash = "sha256:fd313226af375d52e1e36c383f39bf3836e1f192801116b31b090dfcd3ec5266"}, - {file = "mypy-1.12.0.tar.gz", hash = "sha256:65a22d87e757ccd95cbbf6f7e181e6caa87128255eb2b6be901bb71b26d8a99d"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, + {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, + {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, + {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, + {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, + {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, + {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, ] [[package]] @@ -1174,22 +1174,22 @@ files = [ [[package]] name = "orjson" -version = "3.10.7" +version = "3.10.10" requires_python = ">=3.8" summary = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" groups = ["default"] files = [ - {file = "orjson-3.10.7-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:44a96f2d4c3af51bfac6bc4ef7b182aa33f2f054fd7f34cc0ee9a320d051d41f"}, - {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76ac14cd57df0572453543f8f2575e2d01ae9e790c21f57627803f5e79b0d3c3"}, - {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bdbb61dcc365dd9be94e8f7df91975edc9364d6a78c8f7adb69c1cdff318ec93"}, - {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b48b3db6bb6e0a08fa8c83b47bc169623f801e5cc4f24442ab2b6617da3b5313"}, - {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:23820a1563a1d386414fef15c249040042b8e5d07b40ab3fe3efbfbbcbcb8864"}, - {file = "orjson-3.10.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0c6a008e91d10a2564edbb6ee5069a9e66df3fbe11c9a005cb411f441fd2c09"}, - {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d352ee8ac1926d6193f602cbe36b1643bbd1bbcb25e3c1a657a4390f3000c9a5"}, - {file = "orjson-3.10.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d2d9f990623f15c0ae7ac608103c33dfe1486d2ed974ac3f40b693bad1a22a7b"}, - {file = "orjson-3.10.7-cp312-none-win32.whl", hash = "sha256:7c4c17f8157bd520cdb7195f75ddbd31671997cbe10aee559c2d613592e7d7eb"}, - {file = "orjson-3.10.7-cp312-none-win_amd64.whl", hash = "sha256:1d9c0e733e02ada3ed6098a10a8ee0052dd55774de3d9110d29868d24b17faa1"}, - {file = "orjson-3.10.7.tar.gz", hash = "sha256:75ef0640403f945f3a1f9f6400686560dbfb0fb5b16589ad62cd477043c4eee3"}, + {file = "orjson-3.10.10-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8564f48f3620861f5ef1e080ce7cd122ee89d7d6dacf25fcae675ff63b4d6e05"}, + {file = "orjson-3.10.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5bf161a32b479034098c5b81f2608f09167ad2fa1c06abd4e527ea6bf4837a9"}, + {file = "orjson-3.10.10-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:68b65c93617bcafa7f04b74ae8bc2cc214bd5cb45168a953256ff83015c6747d"}, + {file = "orjson-3.10.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8e28406f97fc2ea0c6150f4c1b6e8261453318930b334abc419214c82314f85"}, + {file = "orjson-3.10.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e4d0d9fe174cc7a5bdce2e6c378bcdb4c49b2bf522a8f996aa586020e1b96cee"}, + {file = "orjson-3.10.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3be81c42f1242cbed03cbb3973501fcaa2675a0af638f8be494eaf37143d999"}, + {file = "orjson-3.10.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:65f9886d3bae65be026219c0a5f32dbbe91a9e6272f56d092ab22561ad0ea33b"}, + {file = "orjson-3.10.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:730ed5350147db7beb23ddaf072f490329e90a1d059711d364b49fe352ec987b"}, + {file = "orjson-3.10.10-cp312-none-win32.whl", hash = "sha256:a8f4bf5f1c85bea2170800020d53a8877812892697f9c2de73d576c9307a8a5f"}, + {file = "orjson-3.10.10-cp312-none-win_amd64.whl", hash = "sha256:384cd13579a1b4cd689d218e329f459eb9ddc504fa48c5a83ef4889db7fd7a4f"}, + {file = "orjson-3.10.10.tar.gz", hash = "sha256:37949383c4df7b4337ce82ee35b6d7471e55195efa7dcb45ab8226ceadb0fe3b"}, ] [[package]] @@ -1506,17 +1506,17 @@ files = [ [[package]] name = "pytest-cov" -version = "5.0.0" -requires_python = ">=3.8" +version = "6.0.0" +requires_python = ">=3.9" summary = "Pytest plugin for measuring coverage." groups = ["test"] dependencies = [ - "coverage[toml]>=5.2.1", + "coverage[toml]>=7.5", "pytest>=4.6", ] files = [ - {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, - {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, + {file = "pytest-cov-6.0.0.tar.gz", hash = "sha256:fde0b595ca248bb8e2d76f020b465f3b107c9632e6a1d1705f17834c89dcadc0"}, + {file = "pytest_cov-6.0.0-py3-none-any.whl", hash = "sha256:eee6f1b9e61008bd34975a4d5bab25801eb31898b032dd55addc93e96fcaaa35"}, ] [[package]] @@ -1715,29 +1715,29 @@ files = [ [[package]] name = "ruff" -version = "0.7.0" +version = "0.7.1" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." groups = ["lint"] files = [ - {file = "ruff-0.7.0-py3-none-linux_armv6l.whl", hash = "sha256:0cdf20c2b6ff98e37df47b2b0bd3a34aaa155f59a11182c1303cce79be715628"}, - {file = "ruff-0.7.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:496494d350c7fdeb36ca4ef1c9f21d80d182423718782222c29b3e72b3512737"}, - {file = "ruff-0.7.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:214b88498684e20b6b2b8852c01d50f0651f3cc6118dfa113b4def9f14faaf06"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:630fce3fefe9844e91ea5bbf7ceadab4f9981f42b704fae011bb8efcaf5d84be"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:211d877674e9373d4bb0f1c80f97a0201c61bcd1e9d045b6e9726adc42c156aa"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:194d6c46c98c73949a106425ed40a576f52291c12bc21399eb8f13a0f7073495"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:82c2579b82b9973a110fab281860403b397c08c403de92de19568f32f7178598"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9af971fe85dcd5eaed8f585ddbc6bdbe8c217fb8fcf510ea6bca5bdfff56040e"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b641c7f16939b7d24b7bfc0be4102c56562a18281f84f635604e8a6989948914"}, - {file = "ruff-0.7.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d71672336e46b34e0c90a790afeac8a31954fd42872c1f6adaea1dff76fd44f9"}, - {file = "ruff-0.7.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ab7d98c7eed355166f367597e513a6c82408df4181a937628dbec79abb2a1fe4"}, - {file = "ruff-0.7.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1eb54986f770f49edb14f71d33312d79e00e629a57387382200b1ef12d6a4ef9"}, - {file = "ruff-0.7.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:dc452ba6f2bb9cf8726a84aa877061a2462afe9ae0ea1d411c53d226661c601d"}, - {file = "ruff-0.7.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4b406c2dce5be9bad59f2de26139a86017a517e6bcd2688da515481c05a2cb11"}, - {file = "ruff-0.7.0-py3-none-win32.whl", hash = "sha256:f6c968509f767776f524a8430426539587d5ec5c662f6addb6aa25bc2e8195ec"}, - {file = "ruff-0.7.0-py3-none-win_amd64.whl", hash = "sha256:ff4aabfbaaba880e85d394603b9e75d32b0693152e16fa659a3064a85df7fce2"}, - {file = "ruff-0.7.0-py3-none-win_arm64.whl", hash = "sha256:10842f69c245e78d6adec7e1db0a7d9ddc2fff0621d730e61657b64fa36f207e"}, - {file = "ruff-0.7.0.tar.gz", hash = "sha256:47a86360cf62d9cd53ebfb0b5eb0e882193fc191c6d717e8bef4462bc3b9ea2b"}, + {file = "ruff-0.7.1-py3-none-linux_armv6l.whl", hash = "sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89"}, + {file = "ruff-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35"}, + {file = "ruff-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a"}, + {file = "ruff-0.7.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8"}, + {file = "ruff-0.7.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd"}, + {file = "ruff-0.7.1-py3-none-win32.whl", hash = "sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9"}, + {file = "ruff-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307"}, + {file = "ruff-0.7.1-py3-none-win_arm64.whl", hash = "sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37"}, + {file = "ruff-0.7.1.tar.gz", hash = "sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4"}, ] [[package]] @@ -2003,13 +2003,13 @@ files = [ [[package]] name = "survey" -version = "5.4.0" +version = "5.4.2" requires_python = ">=3.7" summary = "A simple library for creating beautiful interactive prompts." groups = ["default"] files = [ - {file = "survey-5.4.0-py3-none-any.whl", hash = "sha256:45623d45545597cd491690a07a89ff49fc73d4c70a5371e3da776f366dc106b7"}, - {file = "survey-5.4.0.tar.gz", hash = "sha256:253d2d70d05178d75f6d9af04a0bf4385701cfd1e9d4de491789cff685df4c9b"}, + {file = "survey-5.4.2-py3-none-any.whl", hash = "sha256:49473702e82542ea14c9e983db234935a76add4cd5d5437e72b5ea9a5d214c93"}, + {file = "survey-5.4.2.tar.gz", hash = "sha256:e2f9e3b526f0db2fe8f9f4960e5fc2a93119925d2350b18e2d445d6740692af6"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index d6c85e438..814ceaada 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ dependencies = [ "python-dotenv~=1.0", "python-json-logger~=2.0", "ruamel.yaml~=0.18.6", + "scalecodec~=1.2", "sentry-sdk~=2.16", "sqlparse~=0.5", "starknet-py==0.24.0", @@ -79,7 +80,6 @@ dependencies = [ "tortoise-orm==0.21.7", "uvloop~=0.20", "web3~=7.2", - "scalecodec", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index 1b030fcd4..04042bc01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ apscheduler==3.10.4 argcomplete==3.5.1 asgiref==3.8.1 async-lru==2.0.4 -asyncpg==0.29.0 +asyncpg==0.30.0 attrs==24.2.0 base58==2.1.1 bitarray==3.0.0 @@ -56,7 +56,7 @@ mpmath==1.3.0 msgpack==1.1.0 multidict==6.1.0 mypy-extensions==1.0.0 -orjson==3.10.7 +orjson==3.10.10 packaging==24.1 parsimonious==0.10.0 pathspec==0.12.1 @@ -88,7 +88,7 @@ sniffio==1.3.1 sqlparse==0.5.1 starknet-py==0.24.0 strict-rfc3339==0.7 -survey==5.4.0 +survey==5.4.2 sympy==1.11.1 tabulate==0.9.0 toolz==1.0.0; implementation_name == "pypy" or implementation_name == "cpython" diff --git a/src/demo_blank/replay.yaml b/src/demo_blank/replay.yaml deleted file mode 100644 index ac82b11c7..000000000 --- a/src/demo_blank/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Empty config for a fresh start - package: demo_blank - template: demo_blank \ No newline at end of file diff --git a/src/demo_evm_events/replay.yaml b/src/demo_evm_events/replay.yaml deleted file mode 100644 index 1f87ef4b8..000000000 --- a/src/demo_evm_events/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: ERC-20 token transfers (from event logs) - package: demo_evm_events - template: demo_evm_events \ No newline at end of file diff --git a/src/demo_evm_transactions/replay.yaml b/src/demo_evm_transactions/replay.yaml deleted file mode 100644 index 6bc0d88a9..000000000 --- a/src/demo_evm_transactions/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: ERC-20 token transfers (from transactions) - package: demo_evm_transactions - template: demo_evm_transactions \ No newline at end of file diff --git a/src/demo_evm_uniswap/replay.yaml b/src/demo_evm_uniswap/replay.yaml deleted file mode 100644 index 3a6dbc919..000000000 --- a/src/demo_evm_uniswap/replay.yaml +++ /dev/null @@ -1,7 +0,0 @@ -spec_version: 3.0 -replay: - description: Uniswap V3 pools, positions, etc. (advanced, uses TimescaleDB) - package: demo_evm_uniswap - template: demo_evm_uniswap - postgres_image: timescale/timescaledb-ha:pg15 - postgres_data_path: /home/postgres/pgdata/data diff --git a/src/demo_starknet_events/replay.yaml b/src/demo_starknet_events/replay.yaml deleted file mode 100644 index 69baa5ca7..000000000 --- a/src/demo_starknet_events/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: ERC-20 token transfers (from events) - package: demo_starknet_events - template: demo_starknet_events \ No newline at end of file diff --git a/src/demo_substrate_events/deploy/Dockerfile b/src/demo_substrate_events/deploy/Dockerfile index 68422f99a..4c134a235 100644 --- a/src/demo_substrate_events/deploy/Dockerfile +++ b/src/demo_substrate_events/deploy/Dockerfile @@ -1,5 +1,6 @@ -FROM dipdup/dipdup:8 +# FROM dipdup/dipdup:8 # FROM ghcr.io/dipdup-io/dipdup:8 +# FIXME: substrate preview FROM ghcr.io/dipdup-io/dipdup:feat-substrate # COPY --chown=dipdup pyproject.toml README.md . diff --git a/src/demo_substrate_events/replay.yaml b/src/demo_substrate_events/replay.yaml deleted file mode 100644 index 8a79c588a..000000000 --- a/src/demo_substrate_events/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Substrate balance transfers [PREVIEW] - package: demo_substrate_events - template: demo_substrate_events \ No newline at end of file diff --git a/src/demo_tezos_auction/replay.yaml b/src/demo_tezos_auction/replay.yaml deleted file mode 100644 index 0ee223009..000000000 --- a/src/demo_tezos_auction/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: NFT marketplace (TzColors) - package: demo_tezos_auction - template: demo_tezos_auction \ No newline at end of file diff --git a/src/demo_tezos_dao/replay.yaml b/src/demo_tezos_dao/replay.yaml deleted file mode 100644 index dcb52cfe7..000000000 --- a/src/demo_tezos_dao/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: DAO registry (Homebase DAO) - package: demo_tezos_dao - template: demo_tezos_dao \ No newline at end of file diff --git a/src/demo_tezos_dex/replay.yaml b/src/demo_tezos_dex/replay.yaml deleted file mode 100644 index f136ce6e5..000000000 --- a/src/demo_tezos_dex/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: DEX balances and liquidity (Quipuswap) - package: demo_tezos_dex - template: demo_tezos_dex \ No newline at end of file diff --git a/src/demo_tezos_domains/replay.yaml b/src/demo_tezos_domains/replay.yaml deleted file mode 100644 index f4c0f7aed..000000000 --- a/src/demo_tezos_domains/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Domain name service (Tezos Domains) - package: demo_tezos_domains - template: demo_tezos_domains \ No newline at end of file diff --git a/src/demo_tezos_etherlink/replay.yaml b/src/demo_tezos_etherlink/replay.yaml deleted file mode 100644 index 8ba255a8a..000000000 --- a/src/demo_tezos_etherlink/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Etherlink smart rollup transactions - package: demo_tezos_etherlink - template: demo_tezos_etherlink \ No newline at end of file diff --git a/src/demo_tezos_events/replay.yaml b/src/demo_tezos_events/replay.yaml deleted file mode 100644 index a70d728a0..000000000 --- a/src/demo_tezos_events/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Processing contract events - package: demo_tezos_events - template: demo_tezos_events \ No newline at end of file diff --git a/src/demo_tezos_factories/replay.yaml b/src/demo_tezos_factories/replay.yaml deleted file mode 100644 index 453f1ff23..000000000 --- a/src/demo_tezos_factories/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Example of spawning indexes in runtime - package: demo_tezos_factories - template: demo_tezos_factories \ No newline at end of file diff --git a/src/demo_tezos_head/replay.yaml b/src/demo_tezos_head/replay.yaml deleted file mode 100644 index ceafdbec4..000000000 --- a/src/demo_tezos_head/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Processing head block metadata (realtime only) - package: demo_tezos_head - template: demo_tezos_head \ No newline at end of file diff --git a/src/demo_tezos_nft_marketplace/replay.yaml b/src/demo_tezos_nft_marketplace/replay.yaml deleted file mode 100644 index 6e8910678..000000000 --- a/src/demo_tezos_nft_marketplace/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: NFT marketplace (hic at nunc) - package: demo_tezos_nft_marketplace - template: demo_tezos_nft_marketplace \ No newline at end of file diff --git a/src/demo_tezos_raw/replay.yaml b/src/demo_tezos_raw/replay.yaml deleted file mode 100644 index baa226c49..000000000 --- a/src/demo_tezos_raw/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: Process raw operations without filtering and typed payloads - package: demo_tezos_raw - template: demo_tezos_raw \ No newline at end of file diff --git a/src/demo_tezos_token/replay.yaml b/src/demo_tezos_token/replay.yaml deleted file mode 100644 index 35e30be6c..000000000 --- a/src/demo_tezos_token/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: FA1.2 token contract operations - package: demo_tezos_token - template: demo_tezos_token \ No newline at end of file diff --git a/src/demo_tezos_token_balances/replay.yaml b/src/demo_tezos_token_balances/replay.yaml deleted file mode 100644 index e2e8cf841..000000000 --- a/src/demo_tezos_token_balances/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: FA1.2 token balances - package: demo_tezos_token_balances - template: demo_tezos_token_balances \ No newline at end of file diff --git a/src/demo_tezos_token_transfers/replay.yaml b/src/demo_tezos_token_transfers/replay.yaml deleted file mode 100644 index fef854554..000000000 --- a/src/demo_tezos_token_transfers/replay.yaml +++ /dev/null @@ -1,5 +0,0 @@ -spec_version: 3.0 -replay: - description: FA1.2 token transfers - package: demo_tezos_token_transfers - template: demo_tezos_token_transfers \ No newline at end of file diff --git a/src/dipdup/codegen/__init__.py b/src/dipdup/codegen/__init__.py index 1954b99c2..87a8bf320 100644 --- a/src/dipdup/codegen/__init__.py +++ b/src/dipdup/codegen/__init__.py @@ -169,6 +169,12 @@ async def _generate_type(self, schema_path: Path, force: bool) -> None: class_name = self.get_typeclass_name(schema_path) self._logger.info('Generating type `%s`', class_name) output_path.parent.mkdir(parents=True, exist_ok=True) + # TODO: make it configurable + model_type = ( + dmcg.DataModelType.TypingTypedDict + if 'substrate' in str(output_path) + else dmcg.DataModelType.PydanticV2BaseModel + ) dmcg.generate( input_=schema_path, output=output_path, @@ -178,11 +184,7 @@ async def _generate_type(self, schema_path: Path, force: bool) -> None: target_python_version=dmcg.PythonVersion.PY_312, custom_file_header=CODEGEN_HEADER, use_union_operator=True, - output_model_type=( - dmcg.DataModelType.TypingTypedDict - if self.kind == 'substrate' - else dmcg.DataModelType.PydanticV2BaseModel - ), + output_model_type=model_type, use_schema_description=True, ) diff --git a/src/dipdup/context.py b/src/dipdup/context.py index 402490baf..0e3a27a61 100644 --- a/src/dipdup/context.py +++ b/src/dipdup/context.py @@ -736,8 +736,6 @@ def _callback_wrapper(self, module: str) -> Iterator[None]: # NOTE: Do not wrap known errors like ProjectImportError except FrameworkException: raise - # except Exception as e: - # raise CallbackError(module, e) from e def _get_handler(self, name: str, index: str) -> HandlerConfig: try: @@ -882,4 +880,4 @@ def _wrap( @property def is_finalized(self) -> bool: # FIXME: check the datasource - return 1 + return True diff --git a/src/dipdup/datasources/abi_etherscan.py b/src/dipdup/datasources/abi_etherscan.py index 196e2b559..092eb9d2a 100644 --- a/src/dipdup/datasources/abi_etherscan.py +++ b/src/dipdup/datasources/abi_etherscan.py @@ -76,7 +76,7 @@ async def get_abi_failover(self, address: str) -> dict[str, Any]: ) ).text() - regex = r'id="js-copytextarea2(.*)>(\[.*?)\<\/pre' + regex = r'id=["\']js-copytextarea2(.*)>(\[.*?)\<\/pre' if (match := re.search(regex, html)) and (abi := match.group(2)): return cast(dict[str, Any], orjson.loads(abi)) raise DatasourceError('Failed to get ABI', self.name) diff --git a/src/dipdup/datasources/evm_node.py b/src/dipdup/datasources/evm_node.py index 44b8517cc..69814350b 100644 --- a/src/dipdup/datasources/evm_node.py +++ b/src/dipdup/datasources/evm_node.py @@ -128,10 +128,10 @@ async def _emitter_loop(self) -> None: # NOTE: Push rollback to all EVM indexes, but continue processing. if head.level <= known_level: for type_ in ( - SubsquidMessageType.blocks, - SubsquidMessageType.logs, - SubsquidMessageType.traces, - SubsquidMessageType.transactions, + SubsquidMessageType.evm_blocks, + SubsquidMessageType.evm_logs, + SubsquidMessageType.evm_traces, + SubsquidMessageType.evm_transactions, ): await self.emit_rollback( type_, diff --git a/src/dipdup/indexes/_subsquid.py b/src/dipdup/indexes/_subsquid.py new file mode 100644 index 000000000..8042fb15f --- /dev/null +++ b/src/dipdup/indexes/_subsquid.py @@ -0,0 +1,90 @@ +import random +from abc import ABC +from abc import abstractmethod +from typing import TYPE_CHECKING +from typing import Any +from typing import Generic +from typing import TypeVar + +if TYPE_CHECKING: + from dipdup.context import DipDupContext +from dipdup.datasources.evm_node import NODE_LAST_MILE +from dipdup.datasources.evm_node import EvmNodeDatasource +from dipdup.index import Index +from dipdup.index import IndexQueueItemT +from dipdup.performance import metrics + +IndexConfigT = TypeVar('IndexConfigT', bound=Any) +DatasourceT = TypeVar('DatasourceT', bound=Any) + + +class SubsquidIndex( + Generic[IndexConfigT, IndexQueueItemT, DatasourceT], + Index[IndexConfigT, IndexQueueItemT, DatasourceT], + ABC, +): + subsquid_datasources: tuple[Any, ...] + node_datasources: tuple[Any, ...] + + def __init__(self, ctx: 'DipDupContext', config: IndexConfigT, datasources: tuple[DatasourceT, ...]) -> None: + super().__init__(ctx, config, datasources) + self._subsquid_started: bool = False + + @abstractmethod + async def _synchronize_subsquid(self, sync_level: int) -> None: ... + + @abstractmethod + async def _synchronize_node(self, sync_level: int) -> None: ... + + async def _get_node_sync_level( + self, + subsquid_level: int, + index_level: int, + node: EvmNodeDatasource | None = None, + ) -> int | None: + if not self.node_datasources: + return None + node = node or random.choice(self.node_datasources) + + node_sync_level = await node.get_head_level() + subsquid_lag = abs(node_sync_level - subsquid_level) + subsquid_available = subsquid_level - index_level + self._logger.info('Subsquid is %s levels behind; %s available', subsquid_lag, subsquid_available) + if subsquid_available < NODE_LAST_MILE: + return node_sync_level + return None + + async def _synchronize(self, sync_level: int) -> None: + """Fetch event logs via Fetcher and pass to message callback""" + index_level = await self._enter_sync_state(sync_level) + if index_level is None: + return + + levels_left = sync_level - index_level + if levels_left <= 0: + return + + if self.subsquid_datasources: + subsquid_sync_level = await self.subsquid_datasources[0].get_head_level() + metrics._sqd_processor_chain_height = subsquid_sync_level + else: + subsquid_sync_level = 0 + + node_sync_level = await self._get_node_sync_level(subsquid_sync_level, index_level) + + # NOTE: Fetch last blocks from node if there are not enough realtime messages in queue + if node_sync_level: + sync_level = min(sync_level, node_sync_level) + self._logger.debug('Using node datasource; sync level: %s', sync_level) + await self._synchronize_node(sync_level) + else: + sync_level = min(sync_level, subsquid_sync_level) + await self._synchronize_subsquid(sync_level) + + if not self.node_datasources and not self._subsquid_started: + self._subsquid_started = True + self._logger.info('No `evm.node` datasources available; polling Subsquid') + for datasource in self.subsquid_datasources: + await datasource.start() + + await self._exit_sync_state(sync_level) diff --git a/src/dipdup/indexes/evm.py b/src/dipdup/indexes/evm.py index 13bac690f..15c79d37d 100644 --- a/src/dipdup/indexes/evm.py +++ b/src/dipdup/indexes/evm.py @@ -1,6 +1,4 @@ -import random from abc import ABC -from abc import abstractmethod from functools import cache from typing import TYPE_CHECKING from typing import Any @@ -8,23 +6,17 @@ from typing import TypeVar from dipdup.config.evm import EvmContractConfig -from dipdup.datasources.evm_node import NODE_LAST_MILE from dipdup.datasources.evm_node import EvmNodeDatasource from dipdup.datasources.evm_subsquid import EvmSubsquidDatasource from dipdup.exceptions import ConfigurationError -from dipdup.index import Index from dipdup.index import IndexQueueItemT +from dipdup.indexes._subsquid import SubsquidIndex from dipdup.package import DipDupPackage -from dipdup.performance import metrics if TYPE_CHECKING: from dipdup.context import DipDupContext -IndexConfigT = TypeVar('IndexConfigT', bound=Any) -DatasourceT = TypeVar('DatasourceT', bound=Any) - - @cache def get_sighash( package: DipDupPackage, @@ -49,15 +41,15 @@ def get_sighash( raise ConfigurationError('Either `to` or `signature` filters are expected') -# FIXME: Should be subsquid one +IndexConfigT = TypeVar('IndexConfigT', bound=Any) +DatasourceT = TypeVar('DatasourceT', bound=Any) + + class EvmIndex( Generic[IndexConfigT, IndexQueueItemT, DatasourceT], - Index[IndexConfigT, IndexQueueItemT, DatasourceT], + SubsquidIndex[IndexConfigT, IndexQueueItemT, DatasourceT], ABC, ): - subsquid_datasources: tuple[Any, ...] - node_datasources: tuple[Any, ...] - def __init__( self, ctx: 'DipDupContext', @@ -67,64 +59,4 @@ def __init__( super().__init__(ctx, config, datasources) self.subsquid_datasources = tuple(d for d in datasources if isinstance(d, EvmSubsquidDatasource)) self.node_datasources = tuple(d for d in datasources if isinstance(d, EvmNodeDatasource)) - self._subsquid_started: bool = False self._abis = ctx.package._evm_abis - - @abstractmethod - async def _synchronize_subsquid(self, sync_level: int) -> None: ... - - @abstractmethod - async def _synchronize_node(self, sync_level: int) -> None: ... - - async def _get_node_sync_level( - self, - subsquid_level: int, - index_level: int, - node: EvmNodeDatasource | None = None, - ) -> int | None: - if not self.node_datasources: - return None - node = node or random.choice(self.node_datasources) - - node_sync_level = await node.get_head_level() - subsquid_lag = abs(node_sync_level - subsquid_level) - subsquid_available = subsquid_level - index_level - self._logger.info('Subsquid is %s levels behind; %s available', subsquid_lag, subsquid_available) - if subsquid_available < NODE_LAST_MILE: - return node_sync_level - return None - - async def _synchronize(self, sync_level: int) -> None: - """Fetch event logs via Fetcher and pass to message callback""" - index_level = await self._enter_sync_state(sync_level) - if index_level is None: - return - - levels_left = sync_level - index_level - if levels_left <= 0: - return - - if self.subsquid_datasources: - subsquid_sync_level = await self.subsquid_datasources[0].get_head_level() - metrics._sqd_processor_chain_height = subsquid_sync_level - else: - subsquid_sync_level = 0 - - node_sync_level = await self._get_node_sync_level(subsquid_sync_level, index_level) - - # NOTE: Fetch last blocks from node if there are not enough realtime messages in queue - if node_sync_level: - sync_level = min(sync_level, node_sync_level) - self._logger.debug('Using node datasource; sync level: %s', sync_level) - await self._synchronize_node(sync_level) - else: - sync_level = min(sync_level, subsquid_sync_level) - await self._synchronize_subsquid(sync_level) - - if not self.node_datasources and not self._subsquid_started: - self._subsquid_started = True - self._logger.info('No `evm.node` datasources available; polling Subsquid') - for datasource in self.subsquid_datasources: - await datasource.start() - - await self._exit_sync_state(sync_level) diff --git a/src/dipdup/indexes/evm_events/index.py b/src/dipdup/indexes/evm_events/index.py index 55e08befa..2f8c1b30b 100644 --- a/src/dipdup/indexes/evm_events/index.py +++ b/src/dipdup/indexes/evm_events/index.py @@ -22,7 +22,7 @@ class EvmEventsIndex( EvmIndex[EvmEventsIndexConfig, QueueItem, EvmDatasource], - message_type=SubsquidMessageType.logs, + message_type=SubsquidMessageType.evm_logs, ): async def _synchronize_subsquid(self, sync_level: int) -> None: diff --git a/src/dipdup/indexes/evm_transactions/index.py b/src/dipdup/indexes/evm_transactions/index.py index 350517bee..0a717ba33 100644 --- a/src/dipdup/indexes/evm_transactions/index.py +++ b/src/dipdup/indexes/evm_transactions/index.py @@ -21,7 +21,7 @@ class EvmTransactionsIndex( EvmIndex[EvmTransactionsIndexConfig, QueueItem, EvmDatasource], - message_type=SubsquidMessageType.transactions, + message_type=SubsquidMessageType.evm_transactions, ): def _match_level_data(self, handlers: Any, level_data: Any) -> deque[Any]: return match_transactions( diff --git a/src/dipdup/indexes/starknet_events/index.py b/src/dipdup/indexes/starknet_events/index.py index 3fcd975b3..702e0281d 100644 --- a/src/dipdup/indexes/starknet_events/index.py +++ b/src/dipdup/indexes/starknet_events/index.py @@ -25,7 +25,7 @@ class StarknetEventsIndex( StarknetIndex[StarknetEventsIndexConfig, QueueItem, StarknetDatasource], - message_type=SubsquidMessageType.logs, + message_type=SubsquidMessageType.starknet_events, ): def __init__( self, diff --git a/src/dipdup/indexes/substrate.py b/src/dipdup/indexes/substrate.py index 78e7a18a2..ab61c57fb 100644 --- a/src/dipdup/indexes/substrate.py +++ b/src/dipdup/indexes/substrate.py @@ -8,7 +8,7 @@ from dipdup.datasources.substrate_subscan import SubstrateSubscanDatasource from dipdup.datasources.substrate_subsquid import SubstrateSubsquidDatasource from dipdup.index import IndexQueueItemT -from dipdup.indexes.evm import EvmIndex +from dipdup.indexes._subsquid import SubsquidIndex from dipdup.runtimes import SubstrateRuntime SubstrateDatasource = SubstrateSubsquidDatasource | SubstrateSubscanDatasource | SubstrateNodeDatasource @@ -22,8 +22,7 @@ class SubstrateIndex( Generic[IndexConfigT, IndexQueueItemT, DatasourceT], - # FIXME: it's not - EvmIndex[IndexConfigT, IndexQueueItemT, DatasourceT], + SubsquidIndex[IndexConfigT, IndexQueueItemT, DatasourceT], ABC, ): def __init__( @@ -33,6 +32,8 @@ def __init__( datasources: tuple[DatasourceT, ...], ) -> None: super().__init__(ctx, config, datasources) + self.subsquid_datasources = tuple(d for d in datasources if isinstance(d, SubstrateSubsquidDatasource)) + self.node_datasources = tuple(d for d in datasources if isinstance(d, SubstrateNodeDatasource)) self.runtime = SubstrateRuntime( config=config.runtime, package=ctx.package, diff --git a/src/dipdup/models/_subsquid.py b/src/dipdup/models/_subsquid.py index 3568df87c..2ec782e58 100644 --- a/src/dipdup/models/_subsquid.py +++ b/src/dipdup/models/_subsquid.py @@ -6,10 +6,11 @@ class SubsquidMessageType(MessageType, Enum): - blocks = 'blocks' - logs = 'logs' - traces = 'traces' - transactions = 'transactions' + evm_blocks = 'evm_blocks' + evm_logs = 'evm_logs' + evm_traces = 'evm_traces' + evm_transactions = 'evm_transactions' + starknet_events = 'starknet_events' substrate_events = 'substrate_events' diff --git a/src/dipdup/project.py b/src/dipdup/project.py index ae7b7f89a..c65bee047 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -318,7 +318,7 @@ def render_project( _render( answers, template_path=Path(__file__).parent / 'templates' / 'replay.yaml.j2', - output_path=Path(answers['package']) / 'configs' / 'replay.yaml', + output_path=get_package_path(answers['package']) / 'configs' / 'replay.yaml', force=force, ) @@ -382,7 +382,7 @@ def _render_templates( # NOTE: If there are files without .j2 extension, just copy them for path in project_files: - if path.is_dir(): + if path.is_dir() or path.name == 'replay.yaml': continue output_path = Path( get_package_path(answers['package']), diff --git a/src/dipdup/projects/demo_substrate_events/deploy/Dockerfile.j2 b/src/dipdup/projects/demo_substrate_events/deploy/Dockerfile.j2 index 3e60e19fb..e1c666465 100644 --- a/src/dipdup/projects/demo_substrate_events/deploy/Dockerfile.j2 +++ b/src/dipdup/projects/demo_substrate_events/deploy/Dockerfile.j2 @@ -1,5 +1,6 @@ -FROM dipdup/dipdup:8 +# FROM dipdup/dipdup:8 # FROM ghcr.io/dipdup-io/dipdup:8 +# FIXME: substrate preview FROM ghcr.io/dipdup-io/dipdup:feat-substrate # COPY --chown=dipdup pyproject.toml README.md . diff --git a/src/dipdup/projects/demo_substrate_events/handlers/on_transfer.py.j2 b/src/dipdup/projects/demo_substrate_events/handlers/on_transfer.py.j2 index d7cfed4c9..59be86595 100644 --- a/src/dipdup/projects/demo_substrate_events/handlers/on_transfer.py.j2 +++ b/src/dipdup/projects/demo_substrate_events/handlers/on_transfer.py.j2 @@ -43,7 +43,7 @@ async def on_transfer( ctx: HandlerContext, event: SubstrateEvent[AssetsTransferredPayload], ) -> None: - amount = Decimal(event.payload.get('amount') or event.payload['value']) + amount = Decimal(event.payload['amount']) if not amount: return diff --git a/src/dipdup/runtimes.py b/src/dipdup/runtimes.py index 96285f1ec..eaa0d4a3f 100644 --- a/src/dipdup/runtimes.py +++ b/src/dipdup/runtimes.py @@ -11,6 +11,7 @@ from dipdup.config.substrate import SubstrateRuntimeConfig from dipdup.exceptions import FrameworkException from dipdup.package import DipDupPackage +from dipdup.utils import sorted_glob if TYPE_CHECKING: from scalecodec.base import RuntimeConfigurationObject # type: ignore[import-untyped] @@ -59,7 +60,6 @@ def get_event_abi(self, qualname: str) -> dict[str, Any]: pallet, name = qualname.split('.') found = False for item in self._metadata: - # FIXME: double break if found: break if item['name'] != pallet: @@ -83,9 +83,13 @@ def __init__( ) -> None: self._config = config self._package = package - # TODO: unload by LRU? + # TODO: Unload not used self._spec_versions: dict[str, SubstrateSpecVersion] = {} + @property + def abi_path(self) -> Path: + return self._package.abi.joinpath(self._config.name) + @cached_property def runtime_config(self) -> 'RuntimeConfigurationObject': from scalecodec.base import RuntimeConfigurationObject @@ -102,14 +106,15 @@ def get_spec_version(self, name: str) -> SubstrateSpecVersion: if name not in self._spec_versions: _logger.info('loading spec version `%s`', name) try: - metadata = orjson.loads(self._package.abi.joinpath(self._config.name, f'v{name}.json').read_bytes()) + metadata_path = self.abi_path.joinpath(f'v{name}.json') + metadata = orjson.loads(metadata_path.read_bytes()) self._spec_versions[name] = SubstrateSpecVersion( name=f'v{name}', metadata=metadata, ) except FileNotFoundError: # FIXME: Using last known version to help with missing abis - last_known = tuple(self._package.abi.joinpath(self._config.name).glob('v*.json'))[-1].stem + last_known = sorted_glob(self.abi_path, 'v*.json')[-1].stem _logger.info('using last known version `%s`', last_known) self._spec_versions[name] = self.get_spec_version(last_known[1:])