Merge branch 'next' into feat/etherlink

CHANGELOG.md

@@ -8,8 +8,14 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic
 
 ### Added
 
+- evm.node: Requesting logs for multiple blocks at once for EVM nodes to improve performance
+- evm.subsquid: Added metrics for Subsquid Cloud deploy.
 - tezos.tzkt: Added support for Etherlink smart rollups (`sr1…` addresses).
 
+### Security
+
+- deps: Updated PyArrow to 14.0.1 to fix [CVE-2023-47248](https://github.com/advisories/GHSA-5wvp-7f3h-6wmm)
+
 ## [7.1.1] - 2023-11-07
 
 ### Fixed

pyproject.toml

@@ -49,18 +49,17 @@ dependencies = [
     "datamodel-code-generator~=0.22.0",
     "pydantic~=1.10.11",
     "tortoise-orm==0.19.3",
-
     "aiohttp~=3.8",
     "aiolimiter~=1.0",
     "anyio~=3.3",
     "APScheduler~=3.8",
     "async-lru~=2.0",
     "asyncclick~=8.0",
     "eth-abi~=4.0",
-    "lru-dict~=1.2",
+    "lru-dict~=1.3.0",
     "orjson~=3.9",
     "prometheus-client~=0.17",
-    "pyarrow~=12.0",
+    "pyarrow>=14.0.1,<15",
     "pycryptodome~=3.17",
     "pyhumps~=3.0",
     "pysignalr~=0.2",

requirements.dev.txt

@@ -16,7 +16,7 @@ asyncpg==0.28.0
 attrs==23.1.0
 babel==2.12.1
 bitarray==2.8.1
-black==23.10.1
+black==23.11.0
 certifi==2023.7.22
 chardet==5.2.0
 charset-normalizer==3.2.0
@@ -54,7 +54,7 @@ lru-dict==1.3.0
 MarkupSafe==2.1.3
 msgpack==1.0.5
 multidict==6.0.4
-mypy==1.6.1
+mypy==1.7.0
 mypy-extensions==1.0.0
 numpy==1.25.2
 openapi-schema-validator==0.4.4
@@ -70,7 +70,7 @@ pprofile==2.1.0
 prance==23.6.21.0
 prometheus-client==0.18.0
 protobuf==4.24.1
-pyarrow==12.0.1
+pyarrow==14.0.1
 pycryptodome==3.19.0
 pydantic==1.10.13
 Pygments==2.16.1
@@ -83,7 +83,7 @@ pytest==7.4.3
 pytest-aiohttp==1.0.5
 pytest-asyncio==0.21.1
 pytest-cov==4.1.0
-pytest-xdist==3.3.1
+pytest-xdist==3.4.0
 python-dotenv==1.0.0
 pytz==2023.3
 pyunormalize==15.0.0
@@ -94,13 +94,13 @@ rfc3339-validator==0.1.4
 rlp==3.0.0
 ruamel-yaml==0.18.5
 ruamel-yaml-clib==0.2.7
-ruff==0.1.4
+ruff==0.1.5
 sentry-sdk==1.34.0
 setuptools==68.2.2
 six==1.16.0
 sniffio==1.3.0
 snowballstemmer==2.2.0
-sphinx==7.2.6
+Sphinx==7.2.6
 sphinx-click==5.0.1
 sphinx-markdown-builder==0.6.5
 sphinxcontrib-applehelp==1.0.7

requirements.txt

@@ -14,7 +14,7 @@ asyncclick==8.1.3.4
 asyncpg==0.28.0
 attrs==23.1.0
 bitarray==2.8.1
-black==23.10.1
+black==23.11.0
 certifi==2023.7.22
 chardet==5.2.0
 charset-normalizer==3.2.0
@@ -59,7 +59,7 @@ platformdirs==3.10.0
 prance==23.6.21.0
 prometheus-client==0.18.0
 protobuf==4.24.1
-pyarrow==12.0.1
+pyarrow==14.0.1
 pycryptodome==3.19.0
 pydantic==1.10.13
 pyhumps==3.8.0

src/dipdup/config/evm_subsquid_events.py

@@ -68,6 +68,7 @@ class SubsquidEventsIndexConfig(IndexConfig):
     handlers: tuple[SubsquidEventsHandlerConfig, ...] = field(default_factory=tuple)
     abi: AbiDatasourceConfig | tuple[AbiDatasourceConfig, ...] | None = None
     node_only: bool = False
+    expose_metrics: bool = False
 
     first_level: int = 0
     last_level: int = 0

src/dipdup/datasources/evm_node.py

@@ -42,7 +42,7 @@
 HeadCallback = Callable[['EvmNodeDatasource', EvmNodeHeadData], Awaitable[None]]
 LogsCallback = Callable[['EvmNodeDatasource', EvmNodeLogData], Awaitable[None]]
 SyncingCallback = Callable[['EvmNodeDatasource', EvmNodeSyncingData], Awaitable[None]]
-RollbackCallback = Callable[['IndexDatasource', MessageType, int, int], Awaitable[None]]
+RollbackCallback = Callable[['IndexDatasource[Any]', MessageType, int, int], Awaitable[None]]
 
 
 @dataclass

src/dipdup/hasura.py

@@ -60,7 +60,7 @@
     'v1.3.0': 'v1.3.4',
 }
 
-RelationalFieldT = fields.relational.ForeignKeyFieldInstance | fields.relational.ManyToManyFieldInstance
+RelationalFieldT = fields.relational.ForeignKeyFieldInstance[Any] | fields.relational.ManyToManyFieldInstance[Any]
 
 _get_fields_query = """
 query introspectionQuery($name: String!) {

src/dipdup/http.py

@@ -147,6 +147,9 @@ async def _retry_request(
         retry_count = 0 if env.TEST else self._config.retry_count
         retry_count_str = 'inf' if retry_count is sys.maxsize else str(retry_count)
 
+        if Metrics.enabled:
+            Metrics.set_http_errors_in_row(self._url, 0)
+
         while True:
             self._logger.debug('HTTP request attempt %s/%s', attempt, retry_count_str)
             try:
@@ -177,6 +180,10 @@ async def _retry_request(
 
                 self._logger.warning('HTTP request attempt %s/%s failed: %s', attempt, retry_count_str, e)
                 self._logger.info('Waiting %s seconds before retry', ratelimit_sleep or retry_sleep)
+
+                if Metrics.enabled:
+                    Metrics.set_http_errors_in_row(self._url, attempt)
+
                 await asyncio.sleep(ratelimit_sleep or retry_sleep)
                 attempt += 1
                 if not ratelimit_sleep: