From 4922893ad8ac154d8234924d2a33f52faa6de856 Mon Sep 17 00:00:00 2001 From: Lev Gorodetskiy Date: Thu, 19 Dec 2024 17:28:43 -0300 Subject: [PATCH] typing, refs, update head ts --- docs/7.references/3.context.md | 387 ------------------- docs/7.references/4.models.md | 42 -- docs/models.rst | 2 +- scripts/docs.py | 3 +- src/dipdup/config/__init__.py | 3 +- src/dipdup/datasources/__init__.py | 3 +- src/dipdup/datasources/substrate_node.py | 18 +- src/dipdup/datasources/substrate_subsquid.py | 4 +- src/dipdup/dipdup.py | 8 +- src/dipdup/models/substrate.py | 25 +- 10 files changed, 33 insertions(+), 462 deletions(-) diff --git a/docs/7.references/3.context.md b/docs/7.references/3.context.md index fcfe3494d..f48ec1856 100644 --- a/docs/7.references/3.context.md +++ b/docs/7.references/3.context.md @@ -6,390 +6,3 @@ description: "Context reference" # Context reference -
- -## dipdup.context.DipDupContext - -class dipdup.context.DipDupContext(config, package, datasources, transactions) -

Common execution context for handler and hook callbacks.

-
-
Parameters:
-
    -

  • config (DipDupConfig) – DipDup configuration

    • -

  • package (DipDupPackage) – DipDup package

    • -

  • datasources (dict[str, Datasource[Any]]) – Mapping of available datasources

    • -

  • transactions (TransactionManager) – Transaction manager (don’t use it directly)

    • -

  • logger – Context-aware logger instance

    • -
-
-
-
- -
- -## dipdup.context.HandlerContext - -class dipdup.context.HandlerContext(config, package, datasources, transactions, logger, handler_config) -

Execution context of handler callbacks.

-
-
Parameters:
-
    -

  • config (DipDupConfig) – DipDup configuration

    • -

  • package (DipDupPackage) – DipDup package

    • -

  • datasources (dict[str, Datasource[Any]]) – Mapping of available datasources

    • -

  • transactions (TransactionManager) – Transaction manager (don’t use it directly)

    • -

  • logger (Logger) – Context-aware logger instance

    • -

  • handler_config (HandlerConfig) – Configuration of the current handler

    • -
-
-
-
- -
- -## dipdup.context.HookContext - -class dipdup.context.HookContext(config, package, datasources, transactions, logger, hook_config) -

Execution context of hook callbacks.

-
-
Parameters:
-
    -

  • config (DipDupConfig) – DipDup configuration

    • -

  • package (DipDupPackage) – DipDup package

    • -

  • datasources (dict[str, Datasource[Any]]) – Mapping of available datasources

    • -

  • transactions (TransactionManager) – Transaction manager (don’t use it directly)

    • -

  • logger (Logger) – Context-aware logger instance

    • -

  • hook_config (HookConfig) – Configuration of the current hook

    • -
-
-
-
- -
- -## dipdup.context.DipDupContext.add_contract - -async DipDupContext.add_contract(kind, name, address=None, typename=None, code_hash=None) -

Adds contract to the inventory.

-
-
Parameters:
-
    -

  • kind (Literal['tezos', 'evm', 'starknet']) – Either ‘tezos’ or ‘evm’ or ‘starknet’ allowed

    • -

  • name (str) – Contract name

    • -

  • address (str | None) – Contract address

    • -

  • typename (str | None) – Alias for the contract script

    • -

  • code_hash (str | int | None) – Contract code hash

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.add_index - -async DipDupContext.add_index(name, template, values, first_level=0, last_level=0, state=None) -

Adds a new index from template.

-
-
Parameters:
-
    -

  • name (str) – Index name

    • -

  • template (str) – Index template to use

    • -

  • values (dict[str, Any]) – Mapping of values to fill template with

    • -

  • first_level (int) – First level to start indexing from

    • -

  • last_level (int) – Last level to index

    • -

  • state (Index | None) – Initial index state (dev only)

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.execute_sql - -async DipDupContext.execute_sql(name, *args, **kwargs) -

Executes SQL script(s) with given name.

-

If the name path is a directory, all .sql scripts within it will be executed in alphabetical order.

-
-
Parameters:
-
    -

  • name (str) – File or directory within project’s sql directory

    • -

  • args (Any) – Positional arguments to pass to the script

    • -

  • kwargs (Any) – Keyword arguments to pass to the script

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.execute_sql_query - -async DipDupContext.execute_sql_query(name, *args) -

Executes SQL query with given name included with the project

-
-
Parameters:
-
    -

  • name (str) – SQL query name within sql directory

    • -

  • values – Values to pass to the query

    • -

  • args (Any)

    • -
-
-
Return type:
-

Any

-
-
-
- -
- -## dipdup.context.DipDupContext.fire_hook - -async DipDupContext.fire_hook(name, wait=True, **kwargs) -

Fire hook with given name and arguments.

-
-
Parameters:
-
    -

  • name (str) – Hook name

    • -

  • wait (bool) – Wait for hook to finish or fire and forget

    • -

  • kwargs (Any) – Hook arguments

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.get_abi_etherscan_datasource - -DipDupContext.get_abi_etherscan_datasource(name) -

Get abi.etherscan datasource by name

-
-
Parameters:
-

name (str) – Name of the datasource

-
-
Return type:
-

AbiEtherscanDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_coinbase_datasource - -DipDupContext.get_coinbase_datasource(name) -

Get coinbase datasource by name

-
-
Parameters:
-

name (str) – Name of the datasource

-
-
Return type:
-

CoinbaseDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_evm_node_datasource - -DipDupContext.get_evm_node_datasource(name) -

Get evm.node datasource by name

-
-
Parameters:
-

name (str)

-
-
Return type:
-

EvmNodeDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_evm_subsquid_datasource - -DipDupContext.get_evm_subsquid_datasource(name) -

Get evm.subsquid datasource by name

-
-
Parameters:
-

name (str)

-
-
Return type:
-

EvmSubsquidDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_http_datasource - -DipDupContext.get_http_datasource(name) -

Get http datasource by name

-
-
Parameters:
-

name (str) – Name of the datasource

-
-
Return type:
-

HttpDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_ipfs_datasource - -DipDupContext.get_ipfs_datasource(name) -

Get ipfs datasource by name

-
-
Parameters:
-

name (str) – Name of the datasource

-
-
Return type:
-

IpfsDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_tzip_metadata_datasource - -DipDupContext.get_tzip_metadata_datasource(name) -

Get metadata datasource by name

-
-
Parameters:
-

name (str) – Name of the datasource

-
-
Return type:
-

TzipMetadataDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.get_tezos_tzkt_datasource - -DipDupContext.get_tezos_tzkt_datasource(name) -

Get tezos.tzkt datasource by name

-
-
Parameters:
-

name (str)

-
-
Return type:
-

TezosTzktDatasource

-
-
-
- -
- -## dipdup.context.DipDupContext.reindex - -async DipDupContext.reindex(reason=None, **context) -

Drops the entire database and starts the indexing process from scratch.

-
-
Parameters:
-
    -

  • reason (str | ReindexingReason | None) – Reason for reindexing in free-form string

    • -

  • context (Any) – Additional information to include in exception message

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.restart - -async DipDupContext.restart() -

Restart process and continue indexing.

-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.rollback - -async DipDupContext.rollback(index, from_level, to_level) -

Rollback index to a given level reverting all changes made since that level.

-
-
Parameters:
-
    -

  • index (str) – Index name

    • -

  • from_level (int) – Level to rollback from

    • -

  • to_level (int) – Level to rollback to

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.update_contract_metadata - -async DipDupContext.update_contract_metadata(network, address, metadata) -

Inserts or updates corresponding rows in the internal dipdup_contract_metadata table -to provide a generic metadata interface (see docs).

-
-
Parameters:
-
    -

  • network (str) – Network name (e.g. mainnet)

    • -

  • address (str) – Contract address

    • -

  • metadata (dict[str, Any] | None) – Contract metadata to insert/update

    • -
-
-
Return type:
-

None

-
-
-
- -
- -## dipdup.context.DipDupContext.update_token_metadata - -async DipDupContext.update_token_metadata(network, address, token_id, metadata) -

Inserts or updates corresponding rows in the internal dipdup_token_metadata table -to provide a generic metadata interface (see docs).

-
-
Parameters:
-
    -

  • network (str) – Network name (e.g. mainnet)

    • -

  • address (str) – Contract address

    • -

  • token_id (str) – Token ID

    • -

  • metadata (dict[str, Any] | None) – Token metadata to insert/update

    • -
-
-
Return type:
-

None

-
-
-
\ No newline at end of file diff --git a/docs/7.references/4.models.md b/docs/7.references/4.models.md index 7d7c095bf..376b853c6 100644 --- a/docs/7.references/4.models.md +++ b/docs/7.references/4.models.md @@ -399,48 +399,6 @@ description: "Models reference" ## Substrate -
- -## dipdup.models.substrate.SubstrateEvent - -class dipdup.models.substrate.SubstrateEvent(data: dipdup.models.substrate.SubstrateEventData, runtime: dipdup.runtimes.SubstrateRuntime) -
-
Parameters:
-
-
-
-
- -
- -## dipdup.models.substrate.SubstrateEventData - -class dipdup.models.substrate.SubstrateEventData(*, name: str, index: int, extrinsicIndex: int, callAddress: list[str] | None, args: list[dipdup.fields.Any] | None = None, decoded_args: dict[str, dipdup.fields.Any] | None = None, header: dipdup.models.substrate.BlockHeader) -
-
Parameters:
-
    -

  • name (str)

    • -

  • index (int)

    • -

  • extrinsicIndex (int)

    • -

  • callAddress (list[str] | None)

    • -

  • args (list[Any] | None)

    • -

  • decoded_args (dict[str, Any] | None)

    • -

  • header (BlockHeader)

    • -
-
-
-
- -
- -## dipdup.models.substrate.BlockHeader - -class dipdup.models.substrate.BlockHeader -
- diff --git a/docs/models.rst b/docs/models.rst index 546e1a927..cc87b305b 100644 --- a/docs/models.rst +++ b/docs/models.rst @@ -55,7 +55,7 @@ Substrate .. autoclass:: dipdup.models.substrate.SubstrateEvent .. autoclass:: dipdup.models.substrate.SubstrateEventData -.. autoclass:: dipdup.models.substrate.BlockHeader +.. autoclass:: dipdup.models.substrate.SubstrateHeadBlockData ------------------------------------------------------------------------------- Tezos diff --git a/scripts/docs.py b/scripts/docs.py index f832efeb1..c9336f47a 100755 --- a/scripts/docs.py +++ b/scripts/docs.py @@ -145,7 +145,6 @@ class ReferencePage(TypedDict): 'dipdup.models.QuerySet', 'dipdup.models.RollbackMessage', 'dipdup.models.substrate.HeadBlock', - 'dipdup.models.substrate.SubstrateEventDataDict', 'dipdup.models.substrate_node.SubstrateNodeHeadSubscription', 'dipdup.models.subsquid.AbstractSubsquidQuery', 'dipdup.models.subsquid.SubsquidMessageType', @@ -517,7 +516,7 @@ def _compare(ref: str, ignore: set[str]) -> None: else: package_path_str = '.' + package_path.with_suffix('').as_posix().replace('/', '.') # NOTE: Skip private modules and classes - if '._' in package_path_str or 'ABC' in match.group(2): + if '._' in package_path_str or 'ABC' in match.group(2) or match.group(1)[0] == '_': continue classes_in_package.add(f'dipdup.{ref}{package_path_str}.{match.group(1)}') diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index ff73a6525..e4d9d9ea3 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -287,7 +287,8 @@ class DatasourceConfig(ABC, NameMixin): """ kind: str - url: str + url: Url + ws_url: WsUrl | None = None http: HttpConfig | None = None diff --git a/src/dipdup/datasources/__init__.py b/src/dipdup/datasources/__init__.py index 6dbfccbb9..8039f2821 100644 --- a/src/dipdup/datasources/__init__.py +++ b/src/dipdup/datasources/__init__.py @@ -143,8 +143,7 @@ def _get_ws_client(self) -> WebsocketTransport: self._logger.debug('Creating Websocket client') - # FIXME: correct config class - url = self._config.ws_url # type: ignore + url = self._config.ws_url if not url: raise FrameworkException('Spawning node datasource, but `ws_url` is not set') self._ws_client = WebsocketTransport( diff --git a/src/dipdup/datasources/substrate_node.py b/src/dipdup/datasources/substrate_node.py index 62ade832b..2421641e6 100644 --- a/src/dipdup/datasources/substrate_node.py +++ b/src/dipdup/datasources/substrate_node.py @@ -19,10 +19,10 @@ from dipdup.datasources import JsonRpcDatasource from dipdup.exceptions import DatasourceError from dipdup.exceptions import FrameworkException -from dipdup.models.substrate import BlockHeader -from dipdup.models.substrate import HeadBlock from dipdup.models.substrate import SubstrateEventData -from dipdup.models.substrate import SubstrateEventDataDict +from dipdup.models.substrate import SubstrateHeadBlockData +from dipdup.models.substrate import _BlockHeader +from dipdup.models.substrate import _SubstrateNodeEventResponse from dipdup.models.substrate_node import SubstrateNodeHeadSubscription from dipdup.models.substrate_node import SubstrateNodeSubscription from dipdup.pysignalr import Message @@ -32,14 +32,14 @@ _logger = logging.getLogger(__name__) -HeadCallback = Callable[['SubstrateNodeDatasource', HeadBlock], Awaitable[None]] +HeadCallback = Callable[['SubstrateNodeDatasource', SubstrateHeadBlockData], Awaitable[None]] EventCallback = Callable[['SubstrateNodeDatasource', tuple[SubstrateEventData, ...]], Awaitable[None]] # NOTE: Renamed entity class LevelData from evm_node @dataclass class SubscriptionMessage: - head: HeadBlock + head: SubstrateHeadBlockData fetch_events: bool = False @@ -168,7 +168,7 @@ async def subscribe(self) -> None: if isinstance(subscription, SubstrateNodeSubscription): await self._subscribe(subscription) - async def emit_head(self, head: HeadBlock) -> None: + async def emit_head(self, head: SubstrateHeadBlockData) -> None: for fn in self._on_head_callbacks: await fn(self, head) @@ -221,7 +221,7 @@ async def get_head_level(self) -> int: async def get_block_hash(self, height: int) -> str: return await self._jsonrpc_request('chain_getBlockHash', [height]) # type: ignore[no-any-return] - async def get_block_header(self, hash: str) -> BlockHeader: + async def get_block_header(self, hash: str) -> _BlockHeader: response = await self._jsonrpc_request('chain_getHeader', [hash]) # FIXME: missing fields return { @@ -246,10 +246,10 @@ async def get_metadata_header_batch(self, heights: list[int]) -> list[MetadataHe async def get_full_block(self, hash: str) -> dict[str, Any]: return await self._jsonrpc_request('chain_getBlock', [hash]) # type: ignore[no-any-return] - async def get_events(self, block_hash: str) -> tuple[SubstrateEventDataDict, ...]: + async def get_events(self, block_hash: str) -> tuple[_SubstrateNodeEventResponse, ...]: events = await self._interface.get_events(block_hash) - result: list[SubstrateEventDataDict] = [] + result: list[_SubstrateNodeEventResponse] = [] for raw_event in events: event: dict[str, Any] = raw_event.decode() result.append( diff --git a/src/dipdup/datasources/substrate_subsquid.py b/src/dipdup/datasources/substrate_subsquid.py index a6f9b66a8..beadb02ba 100644 --- a/src/dipdup/datasources/substrate_subsquid.py +++ b/src/dipdup/datasources/substrate_subsquid.py @@ -3,7 +3,7 @@ from dipdup.config.substrate_subsquid import SubstrateSubsquidDatasourceConfig from dipdup.datasources._subsquid import AbstractSubsquidDatasource from dipdup.models._subsquid import AbstractSubsquidQuery -from dipdup.models.substrate import SubstrateEventDataSubsquid +from dipdup.models.substrate import _SubstrateSubsquidEventResponse Query = AbstractSubsquidQuery @@ -14,7 +14,7 @@ async def iter_events( first_level: int, last_level: int, names: tuple[str, ...], - ) -> AsyncIterator[tuple[SubstrateEventDataSubsquid, ...]]: + ) -> AsyncIterator[tuple[_SubstrateSubsquidEventResponse, ...]]: current_level = first_level while current_level <= last_level: diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index 859ab6f6f..de8d71aa6 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -74,8 +74,8 @@ from dipdup.models.evm import EvmTransactionData from dipdup.models.evm_node import EvmNodeHeadData from dipdup.models.evm_node import EvmNodeSyncingData -from dipdup.models.substrate import HeadBlock from dipdup.models.substrate import SubstrateEventData +from dipdup.models.substrate import SubstrateHeadBlockData from dipdup.models.tezos import TezosBigMapData from dipdup.models.tezos import TezosEventData from dipdup.models.tezos import TezosHeadBlockData @@ -560,10 +560,10 @@ async def _on_tzkt_events(self, datasource: TezosTzktDatasource, events: tuple[T async def _on_substrate_head( self, datasource: SubstrateNodeDatasource, - head: HeadBlock, + head: SubstrateHeadBlockData, ) -> None: - # TODO: any head updates here? - pass + # TODO: update Head. Does fire_and_forget work atm? + metrics._datasource_head_updated[datasource.name] = time.time() async def _on_substrate_events( self, diff --git a/src/dipdup/models/substrate.py b/src/dipdup/models/substrate.py index 918482a4c..947a892c6 100644 --- a/src/dipdup/models/substrate.py +++ b/src/dipdup/models/substrate.py @@ -11,7 +11,7 @@ from dipdup.runtimes import SubstrateRuntime -class BlockHeaderSubsquid(TypedDict): +class _BlockHeaderExtra(TypedDict): number: int hash: str parentHash: str @@ -26,22 +26,22 @@ class BlockHeaderSubsquid(TypedDict): validator: str -class SubstrateEventDataSubsquid(TypedDict): +class _SubstrateSubsquidEventResponse(TypedDict): name: str index: int extrinsicIndex: int callAddress: list[str] args: list[Any] - header: BlockHeaderSubsquid + header: _BlockHeaderExtra -class BlockHeader(TypedDict): +class _BlockHeader(TypedDict): hash: str number: int prev_root: str -class SubstrateEventDataDict(TypedDict): +class _SubstrateNodeEventResponse(TypedDict): name: str index: int extrinsic_index: int @@ -58,15 +58,15 @@ class SubstrateEventData(HasLevel): # we receive decoded args from node datasource and encoded from subsquid datasource args: list[Any] | None = None decoded_args: dict[str, Any] | None = None - header: BlockHeader - header_extra: BlockHeaderSubsquid | None + header: _BlockHeader + header_extra: _BlockHeaderExtra | None @property def level(self) -> int: # type: ignore[override] return self.header['number'] @classmethod - def from_node(cls, event_dict: SubstrateEventDataDict, header: BlockHeader) -> Self: + def from_node(cls, event_dict: _SubstrateNodeEventResponse, header: _BlockHeader) -> Self: return cls( **event_dict, call_address=None, @@ -76,7 +76,7 @@ def from_node(cls, event_dict: SubstrateEventDataDict, header: BlockHeader) -> S ) @classmethod - def from_subsquid(cls, event_dict: SubstrateEventDataSubsquid) -> Self: + def from_subsquid(cls, event_dict: _SubstrateSubsquidEventResponse) -> Self: return cls( name=event_dict['name'], index=event_dict['index'], @@ -93,7 +93,7 @@ def from_subsquid(cls, event_dict: SubstrateEventDataSubsquid) -> Self: ) -class HeadBlock(TypedDict): +class SubstrateHeadBlockData(TypedDict): parentHash: str number: str stateRoot: str @@ -109,13 +109,14 @@ class SubstrateEvent(Generic[PayloadT]): data: SubstrateEventData runtime: SubstrateRuntime - # TODO: could be used in other models with typed payload + # TODO: Use lazy decoding in other models with typed payload @cached_property def payload(self) -> PayloadT: + # NOTE: from node datasource if self.data.decoded_args is not None: return cast(PayloadT, self.data.decoded_args) - # NOTE: both from subsquid + # NOTE: from subsquid datasource assert self.data.args is not None and self.data.header_extra is not None return cast( PayloadT,