diff --git a/CHANGELOG.md b/CHANGELOG.md index a124f22af..f69154902 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,14 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic ### Added +- evm.node: Requesting logs for multiple blocks at once for EVM nodes to improve performance +- evm.subsquid: Added metrics for Subsquid Cloud deploy. - tezos.tzkt: Added support for Etherlink smart rollups (`sr1…` addresses). +### Security + +- deps: Updated PyArrow to 14.0.1 to fix [CVE-2023-47248](https://github.com/advisories/GHSA-5wvp-7f3h-6wmm) + ## [7.1.1] - 2023-11-07 ### Fixed diff --git a/pdm.lock b/pdm.lock index a28b165e9..6ee1c7cf6 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform"] lock_version = "4.4" -content_hash = "sha256:efee5be5a71d12cb011518dfb65eeafb79905f8b94869c0867032ec6a4fa45c5" +content_hash = "sha256:4d0a9f57ce49b1a3c7b1ed5305dab399487d13dc938f138af8eb8bb9d2ef44cd" [[package]] name = "aiohttp" @@ -235,7 +235,7 @@ files = [ [[package]] name = "black" -version = "23.10.1" +version = "23.11.0" requires_python = ">=3.8" summary = "The uncompromising code formatter." dependencies = [ @@ -246,12 +246,12 @@ dependencies = [ "platformdirs>=2", ] files = [ - {file = "black-23.10.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:b15b75fc53a2fbcac8a87d3e20f69874d161beef13954747e053bca7a1ce53a0"}, - {file = "black-23.10.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:e293e4c2f4a992b980032bbd62df07c1bcff82d6964d6c9496f2cd726e246ace"}, - {file = "black-23.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d56124b7a61d092cb52cce34182a5280e160e6aff3137172a68c2c2c4b76bcb"}, - {file = "black-23.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:3f157a8945a7b2d424da3335f7ace89c14a3b0625e6593d21139c2d8214d55ce"}, - {file = "black-23.10.1-py3-none-any.whl", hash = "sha256:d431e6739f727bb2e0495df64a6c7a5310758e87505f5f8cde9ff6c0f2d7e4fe"}, - {file = "black-23.10.1.tar.gz", hash = "sha256:1f8ce316753428ff68749c65a5f7844631aa18c8679dfd3ca9dc1a289979c258"}, + {file = "black-23.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479"}, + {file = "black-23.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244"}, + {file = "black-23.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221"}, + {file = "black-23.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5"}, + {file = "black-23.11.0-py3-none-any.whl", hash = "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e"}, + {file = "black-23.11.0.tar.gz", hash = "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05"}, ] [[package]] @@ -877,7 +877,7 @@ files = [ [[package]] name = "mypy" -version = "1.6.1" +version = "1.7.0" requires_python = ">=3.8" summary = "Optional static typing for Python" dependencies = [ @@ -885,13 +885,13 @@ dependencies = [ "typing-extensions>=4.1.0", ] files = [ - {file = "mypy-1.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:81af8adaa5e3099469e7623436881eff6b3b06db5ef75e6f5b6d4871263547e5"}, - {file = "mypy-1.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8c223fa57cb154c7eab5156856c231c3f5eace1e0bed9b32a24696b7ba3c3245"}, - {file = "mypy-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8032e00ce71c3ceb93eeba63963b864bf635a18f6c0c12da6c13c450eedb183"}, - {file = "mypy-1.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4c46b51de523817a0045b150ed11b56f9fff55f12b9edd0f3ed35b15a2809de0"}, - {file = "mypy-1.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:19f905bcfd9e167159b3d63ecd8cb5e696151c3e59a1742e79bc3bcb540c42c7"}, - {file = "mypy-1.6.1-py3-none-any.whl", hash = "sha256:4cbe68ef919c28ea561165206a2dcb68591c50f3bcf777932323bc208d949cf1"}, - {file = "mypy-1.6.1.tar.gz", hash = "sha256:4d01c00d09a0be62a4ca3f933e315455bde83f37f892ba4b08ce92f3cf44bcc1"}, + {file = "mypy-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:df67fbeb666ee8828f675fee724cc2cbd2e4828cc3df56703e02fe6a421b7401"}, + {file = "mypy-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a79cdc12a02eb526d808a32a934c6fe6df07b05f3573d210e41808020aed8b5d"}, + {file = "mypy-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65f385a6f43211effe8c682e8ec3f55d79391f70a201575def73d08db68ead1"}, + {file = "mypy-1.7.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e81ffd120ee24959b449b647c4b2fbfcf8acf3465e082b8d58fd6c4c2b27e46"}, + {file = "mypy-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:f29386804c3577c83d76520abf18cfcd7d68264c7e431c5907d250ab502658ee"}, + {file = "mypy-1.7.0-py3-none-any.whl", hash = "sha256:96650d9a4c651bc2a4991cf46f100973f656d69edc7faf91844e87fe627f7e96"}, + {file = "mypy-1.7.0.tar.gz", hash = "sha256:1e280b5697202efa698372d2f39e9a6713a0395a756b1c6bd48995f8d72690dc"}, ] [[package]] @@ -1085,19 +1085,21 @@ files = [ [[package]] name = "pyarrow" -version = "12.0.1" -requires_python = ">=3.7" +version = "14.0.1" +requires_python = ">=3.8" summary = "Python library for Apache Arrow" dependencies = [ "numpy>=1.16.6", ] files = [ - {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, - {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, - {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, - {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, - {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, - {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, + {file = "pyarrow-14.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c7331b4ed3401b7ee56f22c980608cf273f0380f77d0f73dd3c185f78f5a6220"}, + {file = "pyarrow-14.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:922e8b49b88da8633d6cac0e1b5a690311b6758d6f5d7c2be71acb0f1e14cd61"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58c889851ca33f992ea916b48b8540735055201b177cb0dcf0596a495a667b00"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30d8494870d9916bb53b2a4384948491444741cb9a38253c590e21f836b01222"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:be28e1a07f20391bb0b15ea03dcac3aade29fc773c5eb4bee2838e9b2cdde0cb"}, + {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:981670b4ce0110d8dcb3246410a4aabf5714db5d8ea63b15686bce1c914b1f83"}, + {file = "pyarrow-14.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:4756a2b373a28f6166c42711240643fb8bd6322467e9aacabd26b488fa41ec23"}, + {file = "pyarrow-14.0.1.tar.gz", hash = "sha256:b8b3f4fe8d4ec15e1ef9b599b94683c5216adaed78d5cb4c606180546d1e2ee1"}, ] [[package]] @@ -1299,7 +1301,7 @@ files = [ [[package]] name = "pytest-xdist" -version = "3.3.1" +version = "3.4.0" requires_python = ">=3.7" summary = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" dependencies = [ @@ -1307,8 +1309,8 @@ dependencies = [ "pytest>=6.2.0", ] files = [ - {file = "pytest-xdist-3.3.1.tar.gz", hash = "sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93"}, - {file = "pytest_xdist-3.3.1-py3-none-any.whl", hash = "sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2"}, + {file = "pytest-xdist-3.4.0.tar.gz", hash = "sha256:3a94a931dd9e268e0b871a877d09fe2efb6175c2c23d60d56a6001359002b832"}, + {file = "pytest_xdist-3.4.0-py3-none-any.whl", hash = "sha256:e513118bf787677a427e025606f55e95937565e06dfaac8d87f55301e57ae607"}, ] [[package]] @@ -1459,27 +1461,27 @@ files = [ [[package]] name = "ruff" -version = "0.1.4" +version = "0.1.5" requires_python = ">=3.7" summary = "An extremely fast Python linter and code formatter, written in Rust." files = [ - {file = "ruff-0.1.4-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:864958706b669cce31d629902175138ad8a069d99ca53514611521f532d91495"}, - {file = "ruff-0.1.4-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9fdd61883bb34317c788af87f4cd75dfee3a73f5ded714b77ba928e418d6e39e"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4eaca8c9cc39aa7f0f0d7b8fe24ecb51232d1bb620fc4441a61161be4a17539"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a9a1301dc43cbf633fb603242bccd0aaa34834750a14a4c1817e2e5c8d60de17"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78e8db8ab6f100f02e28b3d713270c857d370b8d61871d5c7d1702ae411df683"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:80fea754eaae06335784b8ea053d6eb8e9aac75359ebddd6fee0858e87c8d510"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bc02a480d4bfffd163a723698da15d1a9aec2fced4c06f2a753f87f4ce6969c"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9862811b403063765b03e716dac0fda8fdbe78b675cd947ed5873506448acea4"}, - {file = "ruff-0.1.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58826efb8b3efbb59bb306f4b19640b7e366967a31c049d49311d9eb3a4c60cb"}, - {file = "ruff-0.1.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fdfd453fc91d9d86d6aaa33b1bafa69d114cf7421057868f0b79104079d3e66e"}, - {file = "ruff-0.1.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e8791482d508bd0b36c76481ad3117987301b86072158bdb69d796503e1c84a8"}, - {file = "ruff-0.1.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01206e361021426e3c1b7fba06ddcb20dbc5037d64f6841e5f2b21084dc51800"}, - {file = "ruff-0.1.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:645591a613a42cb7e5c2b667cbefd3877b21e0252b59272ba7212c3d35a5819f"}, - {file = "ruff-0.1.4-py3-none-win32.whl", hash = "sha256:99908ca2b3b85bffe7e1414275d004917d1e0dfc99d497ccd2ecd19ad115fd0d"}, - {file = "ruff-0.1.4-py3-none-win_amd64.whl", hash = "sha256:1dfd6bf8f6ad0a4ac99333f437e0ec168989adc5d837ecd38ddb2cc4a2e3db8a"}, - {file = "ruff-0.1.4-py3-none-win_arm64.whl", hash = "sha256:d98ae9ebf56444e18a3e3652b3383204748f73e247dea6caaf8b52d37e6b32da"}, - {file = "ruff-0.1.4.tar.gz", hash = "sha256:21520ecca4cc555162068d87c747b8f95e1e95f8ecfcbbe59e8dd00710586315"}, + {file = "ruff-0.1.5-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:32d47fc69261c21a4c48916f16ca272bf2f273eb635d91c65d5cd548bf1f3d96"}, + {file = "ruff-0.1.5-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:171276c1df6c07fa0597fb946139ced1c2978f4f0b8254f201281729981f3c17"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ef33cd0bb7316ca65649fc748acc1406dfa4da96a3d0cde6d52f2e866c7b39"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b2c205827b3f8c13b4a432e9585750b93fd907986fe1aec62b2a02cf4401eee6"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb408e3a2ad8f6881d0f2e7ad70cddb3ed9f200eb3517a91a245bbe27101d379"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f20dc5e5905ddb407060ca27267c7174f532375c08076d1a953cf7bb016f5a24"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aafb9d2b671ed934998e881e2c0f5845a4295e84e719359c71c39a5363cccc91"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4894dddb476597a0ba4473d72a23151b8b3b0b5f958f2cf4d3f1c572cdb7af7"}, + {file = "ruff-0.1.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a00a7ec893f665ed60008c70fe9eeb58d210e6b4d83ec6654a9904871f982a2a"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a8c11206b47f283cbda399a654fd0178d7a389e631f19f51da15cbe631480c5b"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fa29e67b3284b9a79b1a85ee66e293a94ac6b7bb068b307a8a373c3d343aa8ec"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9b97fd6da44d6cceb188147b68db69a5741fbc736465b5cea3928fdac0bc1aeb"}, + {file = "ruff-0.1.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:721f4b9d3b4161df8dc9f09aa8562e39d14e55a4dbaa451a8e55bdc9590e20f4"}, + {file = "ruff-0.1.5-py3-none-win32.whl", hash = "sha256:f80c73bba6bc69e4fdc73b3991db0b546ce641bdcd5b07210b8ad6f64c79f1ab"}, + {file = "ruff-0.1.5-py3-none-win_amd64.whl", hash = "sha256:c21fe20ee7d76206d290a76271c1af7a5096bc4c73ab9383ed2ad35f852a0087"}, + {file = "ruff-0.1.5-py3-none-win_arm64.whl", hash = "sha256:82bfcb9927e88c1ed50f49ac6c9728dab3ea451212693fe40d08d314663e412f"}, + {file = "ruff-0.1.5.tar.gz", hash = "sha256:5cbec0ef2ae1748fb194f420fb03fb2c25c3258c86129af7172ff8f198f125ab"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 9e3d396ed..fb609fa22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ dependencies = [ "datamodel-code-generator~=0.22.0", "pydantic~=1.10.11", "tortoise-orm==0.19.3", - "aiohttp~=3.8", "aiolimiter~=1.0", "anyio~=3.3", @@ -57,10 +56,10 @@ dependencies = [ "async-lru~=2.0", "asyncclick~=8.0", "eth-abi~=4.0", - "lru-dict~=1.2", + "lru-dict~=1.3.0", "orjson~=3.9", "prometheus-client~=0.17", - "pyarrow~=12.0", + "pyarrow>=14.0.1,<15", "pycryptodome~=3.17", "pyhumps~=3.0", "pysignalr~=0.2", diff --git a/requirements.dev.txt b/requirements.dev.txt index 344ba1259..b2a97cf34 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -16,7 +16,7 @@ asyncpg==0.28.0 attrs==23.1.0 babel==2.12.1 bitarray==2.8.1 -black==23.10.1 +black==23.11.0 certifi==2023.7.22 chardet==5.2.0 charset-normalizer==3.2.0 @@ -54,7 +54,7 @@ lru-dict==1.3.0 MarkupSafe==2.1.3 msgpack==1.0.5 multidict==6.0.4 -mypy==1.6.1 +mypy==1.7.0 mypy-extensions==1.0.0 numpy==1.25.2 openapi-schema-validator==0.4.4 @@ -70,7 +70,7 @@ pprofile==2.1.0 prance==23.6.21.0 prometheus-client==0.18.0 protobuf==4.24.1 -pyarrow==12.0.1 +pyarrow==14.0.1 pycryptodome==3.19.0 pydantic==1.10.13 Pygments==2.16.1 @@ -83,7 +83,7 @@ pytest==7.4.3 pytest-aiohttp==1.0.5 pytest-asyncio==0.21.1 pytest-cov==4.1.0 -pytest-xdist==3.3.1 +pytest-xdist==3.4.0 python-dotenv==1.0.0 pytz==2023.3 pyunormalize==15.0.0 @@ -94,13 +94,13 @@ rfc3339-validator==0.1.4 rlp==3.0.0 ruamel-yaml==0.18.5 ruamel-yaml-clib==0.2.7 -ruff==0.1.4 +ruff==0.1.5 sentry-sdk==1.34.0 setuptools==68.2.2 six==1.16.0 sniffio==1.3.0 snowballstemmer==2.2.0 -sphinx==7.2.6 +Sphinx==7.2.6 sphinx-click==5.0.1 sphinx-markdown-builder==0.6.5 sphinxcontrib-applehelp==1.0.7 diff --git a/requirements.txt b/requirements.txt index 7a94fb21d..0df59921e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ asyncclick==8.1.3.4 asyncpg==0.28.0 attrs==23.1.0 bitarray==2.8.1 -black==23.10.1 +black==23.11.0 certifi==2023.7.22 chardet==5.2.0 charset-normalizer==3.2.0 @@ -59,7 +59,7 @@ platformdirs==3.10.0 prance==23.6.21.0 prometheus-client==0.18.0 protobuf==4.24.1 -pyarrow==12.0.1 +pyarrow==14.0.1 pycryptodome==3.19.0 pydantic==1.10.13 pyhumps==3.8.0 diff --git a/src/dipdup/config/evm_subsquid_events.py b/src/dipdup/config/evm_subsquid_events.py index 55f2ac254..789740420 100644 --- a/src/dipdup/config/evm_subsquid_events.py +++ b/src/dipdup/config/evm_subsquid_events.py @@ -68,6 +68,7 @@ class SubsquidEventsIndexConfig(IndexConfig): handlers: tuple[SubsquidEventsHandlerConfig, ...] = field(default_factory=tuple) abi: AbiDatasourceConfig | tuple[AbiDatasourceConfig, ...] | None = None node_only: bool = False + expose_metrics: bool = False first_level: int = 0 last_level: int = 0 diff --git a/src/dipdup/datasources/evm_node.py b/src/dipdup/datasources/evm_node.py index b0af3869d..7065f9811 100644 --- a/src/dipdup/datasources/evm_node.py +++ b/src/dipdup/datasources/evm_node.py @@ -42,7 +42,7 @@ HeadCallback = Callable[['EvmNodeDatasource', EvmNodeHeadData], Awaitable[None]] LogsCallback = Callable[['EvmNodeDatasource', EvmNodeLogData], Awaitable[None]] SyncingCallback = Callable[['EvmNodeDatasource', EvmNodeSyncingData], Awaitable[None]] -RollbackCallback = Callable[['IndexDatasource', MessageType, int, int], Awaitable[None]] +RollbackCallback = Callable[['IndexDatasource[Any]', MessageType, int, int], Awaitable[None]] @dataclass diff --git a/src/dipdup/hasura.py b/src/dipdup/hasura.py index f54885dde..ef3a0cdac 100644 --- a/src/dipdup/hasura.py +++ b/src/dipdup/hasura.py @@ -60,7 +60,7 @@ 'v1.3.0': 'v1.3.4', } -RelationalFieldT = fields.relational.ForeignKeyFieldInstance | fields.relational.ManyToManyFieldInstance +RelationalFieldT = fields.relational.ForeignKeyFieldInstance[Any] | fields.relational.ManyToManyFieldInstance[Any] _get_fields_query = """ query introspectionQuery($name: String!) { diff --git a/src/dipdup/http.py b/src/dipdup/http.py index 7e7a7fa82..758c536b0 100644 --- a/src/dipdup/http.py +++ b/src/dipdup/http.py @@ -147,6 +147,9 @@ async def _retry_request( retry_count = 0 if env.TEST else self._config.retry_count retry_count_str = 'inf' if retry_count is sys.maxsize else str(retry_count) + if Metrics.enabled: + Metrics.set_http_errors_in_row(self._url, 0) + while True: self._logger.debug('HTTP request attempt %s/%s', attempt, retry_count_str) try: @@ -177,6 +180,10 @@ async def _retry_request( self._logger.warning('HTTP request attempt %s/%s failed: %s', attempt, retry_count_str, e) self._logger.info('Waiting %s seconds before retry', ratelimit_sleep or retry_sleep) + + if Metrics.enabled: + Metrics.set_http_errors_in_row(self._url, attempt) + await asyncio.sleep(ratelimit_sleep or retry_sleep) attempt += 1 if not ratelimit_sleep: diff --git a/src/dipdup/indexes/evm_subsquid_events/index.py b/src/dipdup/indexes/evm_subsquid_events/index.py index 79fa4bf6a..abd35c6a6 100644 --- a/src/dipdup/indexes/evm_subsquid_events/index.py +++ b/src/dipdup/indexes/evm_subsquid_events/index.py @@ -21,9 +21,11 @@ from dipdup.models.evm_subsquid import SubsquidEventData from dipdup.models.evm_subsquid import SubsquidMessageType from dipdup.performance import metrics +from dipdup.prometheus import Metrics LEVEL_BATCH_TIMEOUT = 1 NODE_SYNC_LIMIT = 128 +NODE_BATCH_SIZE = 100 class SubsquidEventsIndex( @@ -100,6 +102,8 @@ async def _process_queue(self) -> None: for message_level, level_logs in logs_by_level.items(): await self._process_level_events(tuple(level_logs), message_level) + if self._config.expose_metrics: + Metrics.set_sqd_processor_last_block(message_level) def get_sync_level(self) -> int: """Get level index needs to be synchronized to depending on its subscription status""" @@ -131,6 +135,8 @@ async def _synchronize(self, sync_level: int) -> None: return subsquid_sync_level = await self.datasource.get_head_level() + if self._config.expose_metrics: + Metrics.set_sqd_processor_chain_height(subsquid_sync_level) use_node = False if self.node_datasources: @@ -152,28 +158,44 @@ async def _synchronize(self, sync_level: int) -> None: for handler in self._config.handlers: typename = handler.contract.module_name topics.add(self.topics[typename][handler.name]) - # FIXME: This is terribly inefficient (but okay for the last mile); see advanced example in web3.py docs. - for level in range(first_level, sync_level + 1): + + # Requesting blocks info by batch + windows = ((i, min(i + NODE_BATCH_SIZE, sync_level)) for i in range(first_level, sync_level + 1, NODE_BATCH_SIZE + 1)) + for start_level, end_level in windows: # NOTE: Get random one every time + # NOTE: Data for blocks start_level and end_level will be included level_logs = await self.random_node.get_logs( { - 'fromBlock': hex(level), - 'toBlock': hex(level), + 'fromBlock': hex(start_level), + 'toBlock': hex(end_level), } ) - block = await self.random_node.get_block_by_level(level) - if block is None: - raise FrameworkException(f'Block {level} not found') - timestamp = int(block['timestamp'], 16) - parsed_level_logs = tuple(EvmNodeLogData.from_json(log, timestamp) for log in level_logs) - await self._process_level_events(parsed_level_logs, sync_level) + # get timestamps for levels + timestamps = {} + for level in range(start_level, end_level + 1): + block = await self.random_node.get_block_by_level(level) + try: + timestamps[hex(level)] = int(block['timestamp'], 16) + except TypeError as e: + raise FrameworkException(f'Block {level} not found') from e + + # match timestamps with logs + parsed_level_logs = tuple(EvmNodeLogData.from_json(log, timestamps[log['blockNumber']]) + for log in level_logs) + await self._process_level_events(parsed_level_logs, self.topics, sync_level) + + await self._process_level_events(parsed_level_logs, sync_level) + if self._config.expose_metrics: + Metrics.set_sqd_processor_last_block(level) else: sync_level = min(sync_level, subsquid_sync_level) fetcher = self._create_fetcher(first_level, sync_level) async for _level, events in fetcher.fetch_by_level(): await self._process_level_events(events, sync_level) + if self._config.expose_metrics: + Metrics.set_sqd_processor_last_block(_level) await self._exit_sync_state(sync_level) diff --git a/src/dipdup/indexes/tezos_tzkt_operations/matcher.py b/src/dipdup/indexes/tezos_tzkt_operations/matcher.py index 92cf2231a..9f411fe2d 100644 --- a/src/dipdup/indexes/tezos_tzkt_operations/matcher.py +++ b/src/dipdup/indexes/tezos_tzkt_operations/matcher.py @@ -34,7 +34,7 @@ class OperationSubgroup: entrypoints: set[str | None] -OperationsHandlerArgumentU = TzktTransaction | TzktOrigination | TzktOperationData | None +OperationsHandlerArgumentU = TzktTransaction[Any, Any] | TzktOrigination[Any] | TzktOperationData | None MatchedOperationsT = tuple[OperationSubgroup, TzktOperationsHandlerConfigU, deque[OperationsHandlerArgumentU]] diff --git a/src/dipdup/prometheus.py b/src/dipdup/prometheus.py index 8356647a7..87121eed6 100644 --- a/src/dipdup/prometheus.py +++ b/src/dipdup/prometheus.py @@ -61,12 +61,29 @@ 'Number of http errors', ['url', 'status'], ) +_http_errors_in_row = Histogram( + 'dipdup_http_errors_in_row', + """The number of consecutive failed requests""" +) _callback_duration = Histogram( 'dipdup_callback_duration_seconds', 'Duration of callback execution', ['callback'], ) +_sqd_processor_last_block = Gauge( + 'sqd_processor_last_block', + 'The last processed block', +) +_sqd_processor_chain_height = Gauge( + 'sqd_processor_chain_height', + 'Current chain height as reported by the archive', +) +_sqd_processor_archive_http_errors_in_row = Histogram( + 'sqd_processor_archive_http_errors_in_row', + """The number of consecutive failed Archive requests""" +) + class Metrics: enabled = False @@ -133,3 +150,17 @@ def set_levels_to_sync(cls, index: str, levels: int) -> None: @classmethod def set_levels_to_realtime(cls, index: str, levels: int) -> None: _index_levels_to_realtime.labels(index=index).observe(levels) + + @classmethod + def set_sqd_processor_last_block(cls, last_block: int) -> None: + _sqd_processor_last_block.set(last_block) + + @classmethod + def set_sqd_processor_chain_height(cls, chain_height: int) -> None: + _sqd_processor_chain_height.set(chain_height) + + @classmethod + def set_http_errors_in_row(cls, url: str, errors_count: int) -> None: + _http_errors_in_row.observe(errors_count) + if 'subsquid' in url: + _sqd_processor_archive_http_errors_in_row.observe(errors_count)