From 3e6cee8e9c66f4c4b0c0e97e3cee8d16ef957b86 Mon Sep 17 00:00:00 2001 From: Wizard1209 <34334729+Wizard1209@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:34:50 -0300 Subject: [PATCH] Add balances index (#871) Co-authored-by: Vladimir Bobrikov --- CHANGELOG.md | 10 ++ docs/1.getting-started/7.indexes.md | 1 + docs/2.indexes/8.tezos_tzkt_token_balances.md | 19 +++ src/demo_token_balances/.dockerignore | 22 ++++ src/demo_token_balances/.gitignore | 29 +++++ src/demo_token_balances/README.md | 61 ++++++++++ src/demo_token_balances/__init__.py | 0 src/demo_token_balances/abi/.keep | 0 src/demo_token_balances/configs/.keep | 0 .../configs/dipdup.compose.yaml | 22 ++++ .../configs/dipdup.sqlite.yaml | 5 + .../configs/dipdup.swarm.yaml | 22 ++++ src/demo_token_balances/configs/replay.yaml | 15 +++ src/demo_token_balances/deploy/.env.default | 12 ++ src/demo_token_balances/deploy/.keep | 0 src/demo_token_balances/deploy/Dockerfile | 9 ++ .../deploy/compose.sqlite.yaml | 19 +++ .../deploy/compose.swarm.yaml | 92 ++++++++++++++ src/demo_token_balances/deploy/compose.yaml | 55 +++++++++ .../deploy/sqlite.env.default | 5 + .../deploy/swarm.env.default | 12 ++ src/demo_token_balances/dipdup.yaml | 21 ++++ src/demo_token_balances/graphql/.keep | 0 src/demo_token_balances/handlers/.keep | 0 .../handlers/on_balance_update.py | 14 +++ src/demo_token_balances/hasura/.keep | 0 src/demo_token_balances/hooks/.keep | 0 .../hooks/on_index_rollback.py | 16 +++ src/demo_token_balances/hooks/on_reindex.py | 7 ++ src/demo_token_balances/hooks/on_restart.py | 7 ++ .../hooks/on_synchronized.py | 7 ++ src/demo_token_balances/models/.keep | 0 src/demo_token_balances/models/__init__.py | 7 ++ src/demo_token_balances/py.typed | 0 src/demo_token_balances/pyproject.toml | 52 ++++++++ src/demo_token_balances/sql/.keep | 0 .../sql/on_index_rollback/.keep | 0 src/demo_token_balances/sql/on_reindex/.keep | 0 src/demo_token_balances/sql/on_restart/.keep | 0 .../sql/on_synchronized/.keep | 0 src/demo_token_balances/types/.keep | 0 src/dipdup/config/__init__.py | 15 ++- .../config/tezos_tzkt_token_balances.py | 80 ++++++++++++ src/dipdup/context.py | 7 +- src/dipdup/datasources/tezos_tzkt.py | 75 ++++++++++++ .../tezos_tzkt_token_balances/__init__.py | 0 .../tezos_tzkt_token_balances/index.py | 114 ++++++++++++++++++ .../tezos_tzkt_token_balances/matcher.py | 42 +++++++ src/dipdup/models/__init__.py | 1 + src/dipdup/models/tezos_tzkt.py | 77 +++++++++++- .../demo_token_balances/dipdup.yaml.j2 | 21 ++++ .../handlers/on_balance_update.py.j2 | 13 ++ .../demo_token_balances/models/__init__.py.j2 | 8 ++ .../projects/demo_token_balances/replay.yaml | 5 + tests/configs/demo_token_balances.yml | 23 ++++ tests/test_demos.py | 14 +++ 56 files changed, 1031 insertions(+), 5 deletions(-) create mode 100644 docs/2.indexes/8.tezos_tzkt_token_balances.md create mode 100644 src/demo_token_balances/.dockerignore create mode 100644 src/demo_token_balances/.gitignore create mode 100644 src/demo_token_balances/README.md create mode 100644 src/demo_token_balances/__init__.py create mode 100644 src/demo_token_balances/abi/.keep create mode 100644 src/demo_token_balances/configs/.keep create mode 100644 src/demo_token_balances/configs/dipdup.compose.yaml create mode 100644 src/demo_token_balances/configs/dipdup.sqlite.yaml create mode 100644 src/demo_token_balances/configs/dipdup.swarm.yaml create mode 100644 src/demo_token_balances/configs/replay.yaml create mode 100644 src/demo_token_balances/deploy/.env.default create mode 100644 src/demo_token_balances/deploy/.keep create mode 100644 src/demo_token_balances/deploy/Dockerfile create mode 100644 src/demo_token_balances/deploy/compose.sqlite.yaml create mode 100644 src/demo_token_balances/deploy/compose.swarm.yaml create mode 100644 src/demo_token_balances/deploy/compose.yaml create mode 100644 src/demo_token_balances/deploy/sqlite.env.default create mode 100644 src/demo_token_balances/deploy/swarm.env.default create mode 100644 src/demo_token_balances/dipdup.yaml create mode 100644 src/demo_token_balances/graphql/.keep create mode 100644 src/demo_token_balances/handlers/.keep create mode 100644 src/demo_token_balances/handlers/on_balance_update.py create mode 100644 src/demo_token_balances/hasura/.keep create mode 100644 src/demo_token_balances/hooks/.keep create mode 100644 src/demo_token_balances/hooks/on_index_rollback.py create mode 100644 src/demo_token_balances/hooks/on_reindex.py create mode 100644 src/demo_token_balances/hooks/on_restart.py create mode 100644 src/demo_token_balances/hooks/on_synchronized.py create mode 100644 src/demo_token_balances/models/.keep create mode 100644 src/demo_token_balances/models/__init__.py create mode 100644 src/demo_token_balances/py.typed create mode 100644 src/demo_token_balances/pyproject.toml create mode 100644 src/demo_token_balances/sql/.keep create mode 100644 src/demo_token_balances/sql/on_index_rollback/.keep create mode 100644 src/demo_token_balances/sql/on_reindex/.keep create mode 100644 src/demo_token_balances/sql/on_restart/.keep create mode 100644 src/demo_token_balances/sql/on_synchronized/.keep create mode 100644 src/demo_token_balances/types/.keep create mode 100644 src/dipdup/config/tezos_tzkt_token_balances.py create mode 100644 src/dipdup/indexes/tezos_tzkt_token_balances/__init__.py create mode 100644 src/dipdup/indexes/tezos_tzkt_token_balances/index.py create mode 100644 src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py create mode 100644 src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 create mode 100644 src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 create mode 100644 src/dipdup/projects/demo_token_balances/models/__init__.py.j2 create mode 100644 src/dipdup/projects/demo_token_balances/replay.yaml create mode 100644 tests/configs/demo_token_balances.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 451da2348..6ef2825f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning]. +## Unreleased + +### Added + +- tezos.tzkt.token_balances: Added new index. + +### Fixed + +- tezos.tzkt.token_transfers: Fixed token_id handler in token transfers index. + ## [7.0.2] - 2023-10-10 ### Added diff --git a/docs/1.getting-started/7.indexes.md b/docs/1.getting-started/7.indexes.md index b023d6582..ddccbe2c7 100644 --- a/docs/1.getting-started/7.indexes.md +++ b/docs/1.getting-started/7.indexes.md @@ -18,6 +18,7 @@ Multiple indexes are available for different workloads. Every index is linked to | [tezos.tzkt.operations](../2.indexes/5.tezos_tzkt_operations.md) | Tezos | TzKT | typed operations | | [tezos.tzkt.operations_unfiltered](../2.indexes/6.tezos_tzkt_operations_unfiltered.md) | Tezos | TzKT | untyped operations | | [tezos.tzkt.token_transfers](../2.indexes/7.tezos_tzkt_token_transfers.md) | Tezos | TzKT | TZIP-12/16 token transfers | +| [tezos.tzkt.token_balances](../2.indexes/8.tezos_tzkt_token_balances.md) | Tezos | TzKT | TZIP-12/16 token balances | Indexes can join multiple contracts considered as a single application. Also, contracts can be used by multiple indexes of any kind, but make sure that they are independent of each other and that indexed data don't overlap. diff --git a/docs/2.indexes/8.tezos_tzkt_token_balances.md b/docs/2.indexes/8.tezos_tzkt_token_balances.md new file mode 100644 index 000000000..12f0ffa1a --- /dev/null +++ b/docs/2.indexes/8.tezos_tzkt_token_balances.md @@ -0,0 +1,19 @@ +--- +title: "Token balances" +description: "This index allows indexing token balances of contracts compatible with FA1.2 or FA2 standards." +network: "tezos" +--- + +# `tezos.tzkt.token_balances` index + +This index allows indexing token balances of contracts compatible with [FA1.2](https://gitlab.com/tzip/tzip/-/blob/master/proposals/tzip-7/README.md) or [FA2](https://gitlab.com/tzip/tzip/-/blob/master/proposals/tzip-12/tzip-12.md) standards. You can either index transfers and cumulatively calculate balances or use this index type to fetch the latest balance information directly. + +```yaml [dipdup.yaml] +{{ #include ../src/demo_token_balances/dipdup.yaml }} +``` + +Callback receives `TzktTokenBalanceData` model that optionally contains the owner, token, and balance values + +```python +{{ #include ../src/demo_token_balances/handlers/on_balance_update.py }} +``` diff --git a/src/demo_token_balances/.dockerignore b/src/demo_token_balances/.dockerignore new file mode 100644 index 000000000..861e17227 --- /dev/null +++ b/src/demo_token_balances/.dockerignore @@ -0,0 +1,22 @@ +# Ignore all +* + +# Add metadata and build files +!demo_token_balances +!pyproject.toml +!pdm.lock +!README.md + +# Add Python code +!**/*.py +**/.*_cache +**/__pycache__ + +# Add configs and scripts (but not env!) +!**/*.graphql +!**/*.json +!**/*.sql +!**/*.yaml +!**/*.yml +!**/*.j2 +!**/.keep \ No newline at end of file diff --git a/src/demo_token_balances/.gitignore b/src/demo_token_balances/.gitignore new file mode 100644 index 000000000..6961da918 --- /dev/null +++ b/src/demo_token_balances/.gitignore @@ -0,0 +1,29 @@ +# Ignore all +* +!*/ + +# Add metadata and build files +!demo_token_balances +!.gitignore +!.dockerignore +!py.typed +!**/Dockerfile +!**/Makefile +!**/pyproject.toml +!**/pdm.lock +!**/README.md +!**/.keep + +# Add Python code +!**/*.py +**/.*_cache +**/__pycache__ + +# Add configs and scripts (but not env!) +!**/*.graphql +!**/*.json +!**/*.sql +!**/*.yaml +!**/*.yml +!**/*.j2 +!**/*.env.default \ No newline at end of file diff --git a/src/demo_token_balances/README.md b/src/demo_token_balances/README.md new file mode 100644 index 000000000..2caca0555 --- /dev/null +++ b/src/demo_token_balances/README.md @@ -0,0 +1,61 @@ +# demo_token_balances + +TzBTC FA1.2 token balances + +## Installation + +This project is based on [DipDup](https://dipdup.io), a framework for building featureful dapps. + +You need a Linux/macOS system with Python 3.11 installed. Use our installer for easy setup: + +```bash +curl -Lsf https://dipdup.io/install.py | python3 +``` + +See the [Installation](https://dipdup.io/docs/installation) page for all options. + +## Usage + +Run the indexer in-memory: + +```bash +dipdup run +``` + +Store data in SQLite database: + +```bash +dipdup -c . -c configs/dipdup.sqlite.yml run +``` + +Or spawn a docker-compose stack: + +```bash +cp deploy/.env.default deploy/.env +# Edit .env before running +docker-compose -f deploy/compose.yaml up +``` + +## Development setup + +We recommend [PDM](https://pdm.fming.dev/latest/) for managing Python projects. To set up the development environment: + +```bash +pdm install +$(pdm venv activate) +``` + +Some tools are included to help you keep the code quality high: black, ruff and mypy. + +```bash +# Format code +pdm fmt + +# Lint code +pdm lint + +# Build Docker image +pdm image +``` + +Inspect the `pyproject.toml` file. It contains all the dependencies and tools used in the project. \ No newline at end of file diff --git a/src/demo_token_balances/__init__.py b/src/demo_token_balances/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/abi/.keep b/src/demo_token_balances/abi/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/configs/.keep b/src/demo_token_balances/configs/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/configs/dipdup.compose.yaml b/src/demo_token_balances/configs/dipdup.compose.yaml new file mode 100644 index 000000000..98824f36f --- /dev/null +++ b/src/demo_token_balances/configs/dipdup.compose.yaml @@ -0,0 +1,22 @@ +database: + kind: postgres + host: ${POSTGRES_HOST:-db} + port: 5432 + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD} + database: ${POSTGRES_DB:-dipdup} + +hasura: + url: http://${HASURA_HOST:-hasura}:8080 + admin_secret: ${HASURA_SECRET} + allow_aggregations: true + camel_case: true + +sentry: + dsn: ${SENTRY_DSN:-""} + environment: ${SENTRY_ENVIRONMENT:-""} + +prometheus: + host: 0.0.0.0 + +logging: ${LOGLEVEL:-INFO} \ No newline at end of file diff --git a/src/demo_token_balances/configs/dipdup.sqlite.yaml b/src/demo_token_balances/configs/dipdup.sqlite.yaml new file mode 100644 index 000000000..ec7006d3d --- /dev/null +++ b/src/demo_token_balances/configs/dipdup.sqlite.yaml @@ -0,0 +1,5 @@ +database: + kind: sqlite + path: ${SQLITE_PATH:-/tmp/demo_token_balances.sqlite} + +logging: ${LOGLEVEL:-INFO} \ No newline at end of file diff --git a/src/demo_token_balances/configs/dipdup.swarm.yaml b/src/demo_token_balances/configs/dipdup.swarm.yaml new file mode 100644 index 000000000..6c7c79b82 --- /dev/null +++ b/src/demo_token_balances/configs/dipdup.swarm.yaml @@ -0,0 +1,22 @@ +database: + kind: postgres + host: ${POSTGRES_HOST:-demo_token_balances_db} + port: 5432 + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD} + database: ${POSTGRES_DB:-dipdup} + +hasura: + url: http://${HASURA_HOST:-demo_token_balances_hasura}:8080 + admin_secret: ${HASURA_SECRET} + allow_aggregations: false + camel_case: true + +sentry: + dsn: ${SENTRY_DSN:-""} + environment: ${SENTRY_ENVIRONMENT:-""} + +prometheus: + host: 0.0.0.0 + +logging: ${LOGLEVEL:-INFO} \ No newline at end of file diff --git a/src/demo_token_balances/configs/replay.yaml b/src/demo_token_balances/configs/replay.yaml new file mode 100644 index 000000000..ef2c6ef31 --- /dev/null +++ b/src/demo_token_balances/configs/replay.yaml @@ -0,0 +1,15 @@ +# Run `dipdup new --replay configs/replay.yaml` to generate new project from this replay +spec_version: 2.0 +replay: + dipdup_version: 7 + template: demo_token_balances + package: demo_token_balances + version: 0.0.1 + description: TzBTC FA1.2 token balances + license: MIT + name: John Doe + email: john_doe@example.com + postgres_image: postgres:15 + postgres_data_path: /var/lib/postgresql/data + hasura_image: hasura/graphql-engine:latest + line_length: 120 diff --git a/src/demo_token_balances/deploy/.env.default b/src/demo_token_balances/deploy/.env.default new file mode 100644 index 000000000..00b262cb5 --- /dev/null +++ b/src/demo_token_balances/deploy/.env.default @@ -0,0 +1,12 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +HASURA_HOST=hasura +HASURA_SECRET= +LOGLEVEL=INFO +POSTGRES_DB=dipdup +POSTGRES_HOST=db +POSTGRES_PASSWORD= +POSTGRES_USER=dipdup +SENTRY_DSN="" +SENTRY_ENVIRONMENT="" diff --git a/src/demo_token_balances/deploy/.keep b/src/demo_token_balances/deploy/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/deploy/Dockerfile b/src/demo_token_balances/deploy/Dockerfile new file mode 100644 index 000000000..b9215c26f --- /dev/null +++ b/src/demo_token_balances/deploy/Dockerfile @@ -0,0 +1,9 @@ +FROM dipdup/dipdup:7 +# FROM ghcr.io/dipdup-io/dipdup:7 +# FROM ghcr.io/dipdup-io/dipdup:next + +# COPY --chown=dipdup pyproject.toml README.md . +# RUN pip install . + +COPY --chown=dipdup . demo_token_balances +WORKDIR demo_token_balances \ No newline at end of file diff --git a/src/demo_token_balances/deploy/compose.sqlite.yaml b/src/demo_token_balances/deploy/compose.sqlite.yaml new file mode 100644 index 000000000..1c7fc50ef --- /dev/null +++ b/src/demo_token_balances/deploy/compose.sqlite.yaml @@ -0,0 +1,19 @@ +version: "3.8" +name: demo_token_balances + +services: + dipdup: + build: + context: .. + dockerfile: deploy/Dockerfile + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.sqlite.yaml", "run"] + restart: always + env_file: .env + ports: + - 46339 + - 9000 + volumes: + - sqlite:${SQLITE_PATH:-/tmp/demo_token_balances.sqlite} + +volumes: + sqlite: \ No newline at end of file diff --git a/src/demo_token_balances/deploy/compose.swarm.yaml b/src/demo_token_balances/deploy/compose.swarm.yaml new file mode 100644 index 000000000..a64536e89 --- /dev/null +++ b/src/demo_token_balances/deploy/compose.swarm.yaml @@ -0,0 +1,92 @@ +version: "3.8" +name: demo_token_balances + +services: + dipdup: + image: ${IMAGE:-ghcr.io/dipdup-io/dipdup}:${TAG:-7} + depends_on: + - db + - hasura + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.swarm.yaml", "run"] + env_file: .env + networks: + - internal + - prometheus-private + deploy: + mode: replicated + replicas: ${INDEXER_ENABLED:-1} + labels: + - prometheus-job=${SERVICE} + - prometheus-port=8000 + placement: &placement + constraints: + - node.labels.${SERVICE} == true + logging: &logging + driver: "json-file" + options: + max-size: "10m" + max-file: "10" + tag: "\{\{.Name\}\}.\{\{.ImageID\}\}" + + db: + image: postgres:15 + volumes: + - db:/var/lib/postgresql/data + env_file: .env + environment: + - POSTGRES_USER=dipdup + - POSTGRES_DB=dipdup + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - internal + deploy: + mode: replicated + replicas: 1 + placement: *placement + logging: *logging + + hasura: + image: hasura/graphql-engine:latest + depends_on: + - db + environment: + - HASURA_GRAPHQL_DATABASE_URL=postgres://dipdup:${POSTGRES_PASSWORD}@demo_token_balances_db:5432/dipdup + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_SECRET} + - HASURA_GRAPHQL_ENABLE_CONSOLE=true + - HASURA_GRAPHQL_DEV_MODE=false + - HASURA_GRAPHQL_LOG_LEVEL=warn + - HASURA_GRAPHQL_ENABLE_TELEMETRY=false + - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user + - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true + networks: + - internal + - traefik-public + deploy: + mode: replicated + replicas: 1 + labels: + - traefik.enable=true + - traefik.http.services.${SERVICE}.loadbalancer.server.port=8080 + - "traefik.http.routers.${SERVICE}.rule=Host(`${HOST}`) && (PathPrefix(`/v1/graphql`) || PathPrefix(`/api/rest`))" + - traefik.http.routers.${SERVICE}.entrypoints=http,${INGRESS:-ingress} + - "traefik.http.routers.${SERVICE}-console.rule=Host(`${SERVICE}.${SWARM_ROOT_DOMAIN}`)" + - traefik.http.routers.${SERVICE}-console.entrypoints=https + - traefik.http.middlewares.${SERVICE}-console.headers.customrequestheaders.X-Hasura-Admin-Secret=${HASURA_SECRET} + - traefik.http.routers.${SERVICE}-console.middlewares=authelia@docker,${SERVICE}-console + placement: *placement + logging: *logging + +volumes: + db: + +networks: + internal: + traefik-public: + external: true + prometheus-private: + external: true \ No newline at end of file diff --git a/src/demo_token_balances/deploy/compose.yaml b/src/demo_token_balances/deploy/compose.yaml new file mode 100644 index 000000000..884b80f27 --- /dev/null +++ b/src/demo_token_balances/deploy/compose.yaml @@ -0,0 +1,55 @@ +version: "3.8" +name: demo_token_balances + +services: + dipdup: + build: + context: .. + dockerfile: deploy/Dockerfile + restart: always + env_file: .env + ports: + - 46339 + - 9000 + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.compose.yaml", "run"] + depends_on: + - db + - hasura + + db: + image: postgres:15 + ports: + - 5432 + volumes: + - db:/var/lib/postgresql/data + restart: always + env_file: .env + environment: + - POSTGRES_USER=dipdup + - POSTGRES_DB=dipdup + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dipdup"] + interval: 10s + timeout: 5s + retries: 5 + + hasura: + image: hasura/graphql-engine:latest + ports: + - 8080 + depends_on: + - db + restart: always + environment: + - HASURA_GRAPHQL_DATABASE_URL=postgres://dipdup:${POSTGRES_PASSWORD}@db:5432/dipdup + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_SECRET} + - HASURA_GRAPHQL_ENABLE_CONSOLE=true + - HASURA_GRAPHQL_DEV_MODE=true + - HASURA_GRAPHQL_LOG_LEVEL=info + - HASURA_GRAPHQL_ENABLE_TELEMETRY=false + - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user + - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true + +volumes: + db: \ No newline at end of file diff --git a/src/demo_token_balances/deploy/sqlite.env.default b/src/demo_token_balances/deploy/sqlite.env.default new file mode 100644 index 000000000..4cacf6bc5 --- /dev/null +++ b/src/demo_token_balances/deploy/sqlite.env.default @@ -0,0 +1,5 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +LOGLEVEL=INFO +SQLITE_PATH=/tmp/demo_token_balances.sqlite diff --git a/src/demo_token_balances/deploy/swarm.env.default b/src/demo_token_balances/deploy/swarm.env.default new file mode 100644 index 000000000..c4811e380 --- /dev/null +++ b/src/demo_token_balances/deploy/swarm.env.default @@ -0,0 +1,12 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +HASURA_HOST=demo_token_balances_hasura +HASURA_SECRET= +LOGLEVEL=INFO +POSTGRES_DB=dipdup +POSTGRES_HOST=demo_token_balances_db +POSTGRES_PASSWORD= +POSTGRES_USER=dipdup +SENTRY_DSN="" +SENTRY_ENVIRONMENT="" diff --git a/src/demo_token_balances/dipdup.yaml b/src/demo_token_balances/dipdup.yaml new file mode 100644 index 000000000..da40f0441 --- /dev/null +++ b/src/demo_token_balances/dipdup.yaml @@ -0,0 +1,21 @@ +spec_version: 2.0 +package: demo_token_balances + +contracts: + tzbtc_mainnet: + kind: tezos + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: tzbtc + +datasources: + tzkt: + kind: tezos.tzkt + url: https://api.tzkt.io + +indexes: + tzbtc_holders_mainnet: + kind: tezos.tzkt.token_balances + datasource: tzkt + handlers: + - callback: on_balance_update + contract: tzbtc_mainnet \ No newline at end of file diff --git a/src/demo_token_balances/graphql/.keep b/src/demo_token_balances/graphql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/handlers/.keep b/src/demo_token_balances/handlers/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/handlers/on_balance_update.py b/src/demo_token_balances/handlers/on_balance_update.py new file mode 100644 index 000000000..ec6716d4d --- /dev/null +++ b/src/demo_token_balances/handlers/on_balance_update.py @@ -0,0 +1,14 @@ +from decimal import Decimal + +from demo_token_balances import models as models +from dipdup.context import HandlerContext +from dipdup.models.tezos_tzkt import TzktTokenBalanceData + + +async def on_balance_update( + ctx: HandlerContext, + token_balance: TzktTokenBalanceData, +) -> None: + holder, _ = await models.Holder.get_or_create(address=token_balance.contract_address) + holder.balance = Decimal(token_balance.balance_value or 0) / (10**8) + await holder.save() \ No newline at end of file diff --git a/src/demo_token_balances/hasura/.keep b/src/demo_token_balances/hasura/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/hooks/.keep b/src/demo_token_balances/hooks/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/hooks/on_index_rollback.py b/src/demo_token_balances/hooks/on_index_rollback.py new file mode 100644 index 000000000..3d38655e0 --- /dev/null +++ b/src/demo_token_balances/hooks/on_index_rollback.py @@ -0,0 +1,16 @@ +from dipdup.context import HookContext +from dipdup.index import Index + + +async def on_index_rollback( + ctx: HookContext, + index: Index, # type: ignore[type-arg] + from_level: int, + to_level: int, +) -> None: + await ctx.execute_sql('on_index_rollback') + await ctx.rollback( + index=index.name, + from_level=from_level, + to_level=to_level, + ) \ No newline at end of file diff --git a/src/demo_token_balances/hooks/on_reindex.py b/src/demo_token_balances/hooks/on_reindex.py new file mode 100644 index 000000000..0804aae37 --- /dev/null +++ b/src/demo_token_balances/hooks/on_reindex.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_reindex( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_reindex') \ No newline at end of file diff --git a/src/demo_token_balances/hooks/on_restart.py b/src/demo_token_balances/hooks/on_restart.py new file mode 100644 index 000000000..2581b5be3 --- /dev/null +++ b/src/demo_token_balances/hooks/on_restart.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_restart( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_restart') \ No newline at end of file diff --git a/src/demo_token_balances/hooks/on_synchronized.py b/src/demo_token_balances/hooks/on_synchronized.py new file mode 100644 index 000000000..09099e4b6 --- /dev/null +++ b/src/demo_token_balances/hooks/on_synchronized.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_synchronized( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_synchronized') \ No newline at end of file diff --git a/src/demo_token_balances/models/.keep b/src/demo_token_balances/models/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/models/__init__.py b/src/demo_token_balances/models/__init__.py new file mode 100644 index 000000000..833a8deea --- /dev/null +++ b/src/demo_token_balances/models/__init__.py @@ -0,0 +1,7 @@ +from dipdup import fields +from dipdup.models import Model + + +class Holder(Model): + address = fields.TextField(pk=True) + balance = fields.DecimalField(decimal_places=8, max_digits=20, default=0) \ No newline at end of file diff --git a/src/demo_token_balances/py.typed b/src/demo_token_balances/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/pyproject.toml b/src/demo_token_balances/pyproject.toml new file mode 100644 index 000000000..32ad2b1ff --- /dev/null +++ b/src/demo_token_balances/pyproject.toml @@ -0,0 +1,52 @@ +[project] +name = "demo_token_balances" +version = "0.0.1" +description = "TzBTC FA1.2 token balances" +license = { text = "MIT" } +authors = [ + { name = "John Doe", email = "john_doe@example.com" } +] +readme = "README.md" +requires-python = ">=3.11,<3.12" +dependencies = [ + "dipdup>=7,<8" +] + +[tool.pdm.dev-dependencies] +dev = [ + "isort", + "black", + "ruff", + "mypy", +] + +[tool.pdm.scripts] +_isort = "isort ." +_black = "black ." +_ruff = "ruff check --fix ." +_mypy = "mypy --no-incremental --exclude demo_token_balances ." +all = { composite = ["fmt", "lint"] } +fmt = { composite = ["_isort", "_black"] } +lint = { composite = ["_ruff", "_mypy"] } +image = "docker buildx build . --load --progress plain -f deploy/Dockerfile -t demo_token_balances:latest" + +[tool.isort] +line_length = 120 +force_single_line = true + +[tool.black] +line-length = 120 +target-version = ['py311'] +skip-string-normalization = true + +[tool.ruff] +line-length = 120 +target-version = 'py311' + +[tool.mypy] +python_version = "3.11" +plugins = ["pydantic.mypy"] + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" \ No newline at end of file diff --git a/src/demo_token_balances/sql/.keep b/src/demo_token_balances/sql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_index_rollback/.keep b/src/demo_token_balances/sql/on_index_rollback/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_reindex/.keep b/src/demo_token_balances/sql/on_reindex/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_restart/.keep b/src/demo_token_balances/sql/on_restart/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_synchronized/.keep b/src/demo_token_balances/sql/on_synchronized/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/types/.keep b/src/demo_token_balances/types/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 6ec3461a3..8740486aa 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -956,13 +956,20 @@ def _resolve_index_links(self, index_config: ResolvedIndexConfigU) -> None: handler_config.parent = index_config if isinstance(handler_config.contract, str): - handler_config.contract = self.get_contract(handler_config.contract) + handler_config.contract = self.get_tezos_contract(handler_config.contract) if isinstance(handler_config.from_, str): - handler_config.from_ = self.get_contract(handler_config.from_) + handler_config.from_ = self.get_tezos_contract(handler_config.from_) if isinstance(handler_config.to, str): - handler_config.to = self.get_contract(handler_config.to) + handler_config.to = self.get_tezos_contract(handler_config.to) + + elif isinstance(index_config, TzktTokenBalancesIndexConfig): + for handler_config in index_config.handlers: + handler_config.parent = index_config + + if isinstance(handler_config.contract, str): + handler_config.contract = self.get_tezos_contract(handler_config.contract) elif isinstance(index_config, TzktOperationsUnfilteredIndexConfig): index_config.handler_config.parent = index_config @@ -1025,6 +1032,7 @@ def _set_names(self) -> None: from dipdup.config.tezos_tzkt_operations import OperationsHandlerTransactionPatternConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsIndexConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsUnfilteredIndexConfig +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesIndexConfig from dipdup.config.tezos_tzkt_token_transfers import TzktTokenTransfersIndexConfig from dipdup.config.tzip_metadata import TzipMetadataDatasourceConfig @@ -1048,6 +1056,7 @@ def _set_names(self) -> None: | TzktOperationsIndexConfig | TzktOperationsUnfilteredIndexConfig | TzktTokenTransfersIndexConfig + | TzktTokenBalancesIndexConfig ) IndexConfigU = ResolvedIndexConfigU | IndexTemplateConfig diff --git a/src/dipdup/config/tezos_tzkt_token_balances.py b/src/dipdup/config/tezos_tzkt_token_balances.py new file mode 100644 index 000000000..3bb3fb754 --- /dev/null +++ b/src/dipdup/config/tezos_tzkt_token_balances.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Literal + +from pydantic.dataclasses import dataclass +from pydantic.fields import Field + +from dipdup.config import ContractConfig +from dipdup.config import HandlerConfig +from dipdup.config.tezos import TezosContractConfig +from dipdup.config.tezos_tzkt import TzktDatasourceConfig +from dipdup.config.tezos_tzkt import TzktIndexConfig +from dipdup.models.tezos_tzkt import TokenBalanceSubscription + +if TYPE_CHECKING: + from collections.abc import Iterator + + from dipdup.subscriptions import Subscription + + +@dataclass +class TzktTokenBalancesHandlerConfig(HandlerConfig): + """Token balance handler config + + :param callback: Callback name + :param contract: Filter by contract + :param token_id: Filter by token ID + """ + + contract: TezosContractConfig | None = None + token_id: int | None = None + + def iter_imports(self, package: str) -> Iterator[tuple[str, str]]: + """This iterator result will be used in codegen to generate handler(s) template""" + yield 'dipdup.context', 'HandlerContext' + yield 'dipdup.models.tezos_tzkt', 'TzktTokenBalanceData' + yield package, 'models as models' + + def iter_arguments(self) -> Iterator[tuple[str, str]]: + """This iterator result will be used in codegen to generate handler(s) template""" + yield 'ctx', 'HandlerContext' + yield 'token_balance', 'TzktTokenBalanceData' + + +@dataclass +class TzktTokenBalancesIndexConfig(TzktIndexConfig): + """Token balance index config + + :param kind: always `tezos.tzkt.token_balances` + :param datasource: Index datasource to use + :param handlers: Mapping of token transfer handlers + + :param first_level: Level to start indexing from + :param last_level: Level to stop indexing at + """ + + kind: Literal['tezos.tzkt.token_balances'] + datasource: TzktDatasourceConfig + handlers: tuple[TzktTokenBalancesHandlerConfig, ...] = Field(default_factory=tuple) + + first_level: int = 0 + last_level: int = 0 + + def get_subscriptions(self) -> set[Subscription]: + subs = super().get_subscriptions() + if self.datasource.merge_subscriptions: + subs.add(TokenBalanceSubscription()) + else: + for handler_config in self.handlers: + contract = ( + handler_config.contract.address if isinstance(handler_config.contract, ContractConfig) else None + ) + subs.add( + TokenBalanceSubscription( + contract=contract, + token_id=handler_config.token_id, + ) + ) + return subs diff --git a/src/dipdup/context.py b/src/dipdup/context.py index 1407e6116..32e24a810 100644 --- a/src/dipdup/context.py +++ b/src/dipdup/context.py @@ -33,6 +33,7 @@ from dipdup.config.tezos_tzkt_head import TzktHeadIndexConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsIndexConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsUnfilteredIndexConfig +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesIndexConfig from dipdup.config.tezos_tzkt_token_transfers import TzktTokenTransfersIndexConfig from dipdup.database import execute_sql from dipdup.database import execute_sql_query @@ -297,10 +298,11 @@ async def _spawn_index(self, name: str, state: Index | None = None) -> Any: from dipdup.indexes.tezos_tzkt_events.index import TzktEventsIndex from dipdup.indexes.tezos_tzkt_head.index import TzktHeadIndex from dipdup.indexes.tezos_tzkt_operations.index import TzktOperationsIndex + from dipdup.indexes.tezos_tzkt_token_balances.index import TzktTokenBalancesIndex from dipdup.indexes.tezos_tzkt_token_transfers.index import TzktTokenTransfersIndex index_config = cast(ResolvedIndexConfigU, self.config.get_index(name)) - index: TzktOperationsIndex | TzktBigMapsIndex | TzktHeadIndex | TzktTokenTransfersIndex | TzktEventsIndex | SubsquidEventsIndex + index: TzktOperationsIndex | TzktBigMapsIndex | TzktHeadIndex | TzktTokenBalancesIndex | TzktTokenTransfersIndex | TzktEventsIndex | SubsquidEventsIndex datasource_name = index_config.datasource.name datasource: TzktDatasource | SubsquidDatasource @@ -315,6 +317,9 @@ async def _spawn_index(self, name: str, state: Index | None = None) -> Any: elif isinstance(index_config, TzktHeadIndexConfig): datasource = self.get_tzkt_datasource(datasource_name) index = TzktHeadIndex(self, index_config, datasource) + elif isinstance(index_config, TzktTokenBalancesIndexConfig): + datasource = self.get_tzkt_datasource(datasource_name) + index = TzktTokenBalancesIndex(self, index_config, datasource) elif isinstance(index_config, TzktTokenTransfersIndexConfig): datasource = self.get_tzkt_datasource(datasource_name) index = TzktTokenTransfersIndex(self, index_config, datasource) diff --git a/src/dipdup/datasources/tezos_tzkt.py b/src/dipdup/datasources/tezos_tzkt.py index 00fd4d479..72bbebcf3 100644 --- a/src/dipdup/datasources/tezos_tzkt.py +++ b/src/dipdup/datasources/tezos_tzkt.py @@ -43,6 +43,7 @@ from dipdup.models.tezos_tzkt import TzktQuoteData from dipdup.models.tezos_tzkt import TzktRollbackMessage from dipdup.models.tezos_tzkt import TzktSubscription +from dipdup.models.tezos_tzkt import TzktTokenBalanceData from dipdup.models.tezos_tzkt import TzktTokenTransferData from dipdup.utils import split_by_chunks @@ -111,6 +112,18 @@ 'originationId', 'migrationId', ) +TOKEN_BALANCE_FIELDS = ( + 'id', + 'transfersCount', + 'firstLevel', + 'firstTime', + 'lastLevel', + 'lastTime', + 'account', + 'token', + 'balance', + 'balanceValue', +) EVENT_FIELDS = ( 'id', 'level', @@ -127,6 +140,7 @@ HeadCallback = Callable[['TzktDatasource', TzktHeadBlockData], Awaitable[None]] OperationsCallback = Callable[['TzktDatasource', tuple[TzktOperationData, ...]], Awaitable[None]] TokenTransfersCallback = Callable[['TzktDatasource', tuple[TzktTokenTransferData, ...]], Awaitable[None]] +TokenBalancesCallback = Callable[['TzktDatasource', tuple[TzktTokenBalanceData, ...]], Awaitable[None]] BigMapsCallback = Callable[['TzktDatasource', tuple[TzktBigMapData, ...]], Awaitable[None]] EventsCallback = Callable[['TzktDatasource', tuple[TzktEventData, ...]], Awaitable[None]] RollbackCallback = Callable[['TzktDatasource', MessageType, int, int], Awaitable[None]] @@ -236,6 +250,7 @@ def __init__( self._on_head_callbacks: set[HeadCallback] = set() self._on_operations_callbacks: set[OperationsCallback] = set() self._on_token_transfers_callbacks: set[TokenTransfersCallback] = set() + self._on_token_balances_callbacks: set[TokenBalancesCallback] = set() self._on_big_maps_callbacks: set[BigMapsCallback] = set() self._on_events_callbacks: set[EventsCallback] = set() self._on_rollback_callbacks: set[RollbackCallback] = set() @@ -341,6 +356,10 @@ async def emit_token_transfers(self, token_transfers: tuple[TzktTokenTransferDat for fn in self._on_token_transfers_callbacks: await fn(self, token_transfers) + async def emit_token_balances(self, token_balances: tuple[TzktTokenBalanceData, ...]) -> None: + for fn in self._on_token_balances_callbacks: + await fn(self, token_balances) + async def emit_big_maps(self, big_maps: tuple[TzktBigMapData, ...]) -> None: for fn in self._on_big_maps_callbacks: await fn(self, big_maps) @@ -897,6 +916,48 @@ async def iter_token_transfers( ): yield batch + async def get_token_balances( + self, + token_addresses: set[str], + token_ids: set[int], + first_level: int | None = None, + last_level: int | None = None, + offset: int | None = None, + limit: int | None = None, + ) -> tuple[TzktTokenBalanceData, ...]: + params = self._get_request_params( + first_level, + last_level, + offset=offset or 0, + limit=limit, + select=TOKEN_BALANCE_FIELDS, + values=True, + cursor=True, + **{ + 'token.contract.in': ','.join(token_addresses), + 'token.id.in': ','.join(str(token_id) for token_id in token_ids), + }, + ) + raw_token_balances = await self._request_values_dict('get', url='v1/tokens/balances', params=params) + return tuple(TzktTokenBalanceData.from_json(item) for item in raw_token_balances) + + async def iter_token_balances( + self, + token_addresses: set[str], + token_ids: set[int], + first_level: int | None = None, + last_level: int | None = None, + ) -> AsyncIterator[tuple[TzktTokenBalanceData, ...]]: + async for batch in self._iter_batches( + self.get_token_balances, + token_addresses, + token_ids, + first_level, + last_level, + cursor=True, + ): + yield batch + async def get_events( self, addresses: set[str], @@ -1073,6 +1134,7 @@ def _get_signalr_client(self) -> SignalRClient: self._signalr_client.on('operations', partial(self._on_message, TzktMessageType.operation)) self._signalr_client.on('transfers', partial(self._on_message, TzktMessageType.token_transfer)) + self._signalr_client.on('balances', partial(self._on_message, TzktMessageType.token_balance)) self._signalr_client.on('bigmaps', partial(self._on_message, TzktMessageType.big_map)) self._signalr_client.on('head', partial(self._on_message, TzktMessageType.head)) self._signalr_client.on('events', partial(self._on_message, TzktMessageType.event)) @@ -1146,6 +1208,8 @@ async def _on_message(self, type_: TzktMessageType, message: list[dict[str, Any] await self._process_operations_data(cast(list[dict[str, Any]], buffered_message.data)) elif buffered_message.type == TzktMessageType.token_transfer: await self._process_token_transfers_data(cast(list[dict[str, Any]], buffered_message.data)) + elif buffered_message.type == TzktMessageType.token_balance: + await self._process_token_balances_data(cast(list[dict[str, Any]], buffered_message.data)) elif buffered_message.type == TzktMessageType.big_map: await self._process_big_maps_data(cast(list[dict[str, Any]], buffered_message.data)) elif buffered_message.type == TzktMessageType.head: @@ -1182,6 +1246,17 @@ async def _process_token_transfers_data(self, data: list[dict[str, Any]]) -> Non for _level, token_transfers in level_token_transfers.items(): await self.emit_token_transfers(tuple(token_transfers)) + async def _process_token_balances_data(self, data: list[dict[str, Any]]) -> None: + """Parse and emit raw token balances from WS""" + level_token_balances: defaultdict[int, deque[TzktTokenBalanceData]] = defaultdict(deque) + + for token_balance_json in data: + token_balance = TzktTokenBalanceData.from_json(token_balance_json) + level_token_balances[token_balance.level].append(token_balance) + + for _level, token_balances in level_token_balances.items(): + await self.emit_token_balances(tuple(token_balances)) + async def _process_big_maps_data(self, data: list[dict[str, Any]]) -> None: """Parse and emit raw big map diffs from WS""" level_big_maps: defaultdict[int, deque[TzktBigMapData]] = defaultdict(deque) diff --git a/src/dipdup/indexes/tezos_tzkt_token_balances/__init__.py b/src/dipdup/indexes/tezos_tzkt_token_balances/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/dipdup/indexes/tezos_tzkt_token_balances/index.py b/src/dipdup/indexes/tezos_tzkt_token_balances/index.py new file mode 100644 index 000000000..710adf028 --- /dev/null +++ b/src/dipdup/indexes/tezos_tzkt_token_balances/index.py @@ -0,0 +1,114 @@ +from contextlib import ExitStack + +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesHandlerConfig +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesIndexConfig +from dipdup.datasources.tezos_tzkt import TzktDatasource +from dipdup.exceptions import ConfigInitializationException +from dipdup.exceptions import FrameworkException +from dipdup.index import Index +from dipdup.indexes.tezos_tzkt_token_balances.matcher import match_token_balances +from dipdup.models.tezos_tzkt import TzktMessageType +from dipdup.models.tezos_tzkt import TzktRollbackMessage +from dipdup.models.tezos_tzkt import TzktTokenBalanceData +from dipdup.prometheus import Metrics + +TokenBalanceQueueItem = tuple[TzktTokenBalanceData, ...] | TzktRollbackMessage + + +class TzktTokenBalancesIndex( + Index[TzktTokenBalancesIndexConfig, TokenBalanceQueueItem, TzktDatasource], + message_type=TzktMessageType.token_balance, +): + def push_token_balances(self, token_balances: TokenBalanceQueueItem) -> None: + self.push_realtime_message(token_balances) + + async def _synchronize(self, sync_level: int) -> None: + await self._enter_sync_state(sync_level) + await self._synchronize_actual(sync_level) + await self._exit_sync_state(sync_level) + + async def _synchronize_actual(self, head_level: int) -> None: + """Retrieve data for the current level""" + # TODO: think about logging and metrics + + addresses, token_ids = set(), set() + for handler in self._config.handlers: + if handler.contract and handler.contract.address is not None: + addresses.add(handler.contract.address) + if handler.token_id is not None: + token_ids.add(handler.token_id) + + async with self._ctx.transactions.in_transaction(head_level, head_level, self.name): + # NOTE: If index is out of date fetch balances as of the current head. + async for balances_batch in self._datasource.iter_token_balances( + addresses, token_ids, last_level=head_level + ): + matched_handlers = match_token_balances(self._config.handlers, balances_batch) + for handler_config, matched_balance_data in matched_handlers: + await self._call_matched_handler(handler_config, matched_balance_data) + + await self._update_state(level=head_level) + + async def _process_level_token_balances( + self, + token_balances: tuple[TzktTokenBalanceData, ...], + sync_level: int, + ) -> None: + if not token_balances: + return + + batch_level = token_balances[0].level + index_level = self.state.level + if batch_level <= index_level: + raise FrameworkException(f'Batch level is lower than index level: {batch_level} <= {index_level}') + + self._logger.debug('Processing token balances of level %s', batch_level) + matched_handlers = match_token_balances(self._config.handlers, token_balances) + + if Metrics.enabled: + Metrics.set_index_handlers_matched(len(matched_handlers)) + + # NOTE: We still need to bump index level but don't care if it will be done in existing transaction + if not matched_handlers: + await self._update_state(level=batch_level) + return + + async with self._ctx.transactions.in_transaction(batch_level, sync_level, self.name): + for handler_config, token_balance in matched_handlers: + await self._call_matched_handler(handler_config, token_balance) + await self._update_state(level=batch_level) + + async def _call_matched_handler( + self, handler_config: TzktTokenBalancesHandlerConfig, token_balance: TzktTokenBalanceData + ) -> None: + if not handler_config.parent: + raise ConfigInitializationException + + await self._ctx.fire_handler( + handler_config.callback, + handler_config.parent.name, + self.datasource, + # NOTE: missing `operation_id` field in API to identify operation + None, + token_balance, + ) + + async def _process_queue(self) -> None: + """Process WebSocket queue""" + if self._queue: + self._logger.debug('Processing websocket queue') + while self._queue: + message = self._queue.popleft() + if isinstance(message, TzktRollbackMessage): + await self._tzkt_rollback(message.from_level, message.to_level) + continue + + message_level = message[0].level + if message_level <= self.state.level: + self._logger.debug('Skipping outdated message: %s <= %s', message_level, self.state.level) + continue + + with ExitStack() as stack: + if Metrics.enabled: + stack.enter_context(Metrics.measure_level_realtime_duration()) + await self._process_level_token_balances(message, message_level) diff --git a/src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py b/src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py new file mode 100644 index 000000000..46b00a015 --- /dev/null +++ b/src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py @@ -0,0 +1,42 @@ +import logging +from collections import deque +from collections.abc import Iterable + +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesHandlerConfig +from dipdup.models.tezos_tzkt import TzktTokenBalanceData + +_logger = logging.getLogger('dipdup.matcher') + +MatchedTokenBalancesT = tuple[TzktTokenBalancesHandlerConfig, TzktTokenBalanceData] + + +def match_token_balance( + handler_config: TzktTokenBalancesHandlerConfig, + token_balance: TzktTokenBalanceData, +) -> bool: + """Match single token balance with pattern""" + if handler_config.contract: + if handler_config.contract.address != token_balance.contract_address: + return False + if handler_config.token_id is not None: + if handler_config.token_id != token_balance.token_id: + return False + return True + + +def match_token_balances( + handlers: Iterable[TzktTokenBalancesHandlerConfig], token_balances: Iterable[TzktTokenBalanceData] +) -> deque[MatchedTokenBalancesT]: + """Try to match token balances with all index handlers.""" + + matched_handlers: deque[MatchedTokenBalancesT] = deque() + + for token_balance in token_balances: + for handler_config in handlers: + token_balance_matched = match_token_balance(handler_config, token_balance) + if not token_balance_matched: + continue + _logger.debug('%s: `%s` handler matched!', token_balance.level, handler_config.callback) + matched_handlers.append((handler_config, token_balance)) + + return matched_handlers diff --git a/src/dipdup/models/__init__.py b/src/dipdup/models/__init__.py index 85997fd3f..519188a48 100644 --- a/src/dipdup/models/__init__.py +++ b/src/dipdup/models/__init__.py @@ -53,6 +53,7 @@ class IndexType(Enum): tezos_tzkt_big_maps = 'tezos.tzkt.big_maps' tezos_tzkt_head = 'tezos.tzkt.head' tezos_tzkt_token_transfers = 'tezos.tzkt.token_transfers' + tezos_tzkt_token_balances = 'tezos.tzkt.token_balances' tezos_tzkt_events = 'tezos.tzkt.events' evm_subsquid_events = 'evm.subsquid.events' diff --git a/src/dipdup/models/tezos_tzkt.py b/src/dipdup/models/tezos_tzkt.py index dd9085180..b63a9679a 100644 --- a/src/dipdup/models/tezos_tzkt.py +++ b/src/dipdup/models/tezos_tzkt.py @@ -48,6 +48,7 @@ class TzktMessageType(MessageType, Enum): big_map = 'big_map' head = 'head' token_transfer = 'token_transfer' + token_balance = 'token_balance' event = 'event' @@ -119,7 +120,7 @@ class TokenTransferSubscription(TzktSubscription): def get_request(self) -> list[dict[str, Any]]: request: dict[str, Any] = {} if self.token_id: - request['token_id'] = self.token_id + request['tokenId'] = self.token_id if self.contract: request['contract'] = self.contract if self.from_: @@ -129,6 +130,22 @@ def get_request(self) -> list[dict[str, Any]]: return [request] +@dataclass(frozen=True) +class TokenBalanceSubscription(TzktSubscription): + type: Literal['token_balance'] = 'token_balance' + method: Literal['SubscribeToTokenBalances'] = 'SubscribeToTokenBalances' + contract: str | None = None + token_id: int | None = None + + def get_request(self) -> list[dict[str, Any]]: + request: dict[str, Any] = {} + if self.token_id: + request['tokenId'] = self.token_id + if self.contract: + request['contract'] = self.contract + return [request] + + @dataclass(frozen=True) class EventSubscription(TzktSubscription): type: Literal['event'] = 'event' @@ -537,6 +554,64 @@ def from_json(cls, token_transfer_json: dict[str, Any]) -> 'TzktTokenTransferDat ) +@dataclass(frozen=True) +class TzktTokenBalanceData(HasLevel): + """Basic structure for token transver received from TzKT SignalR API""" + + id: int + transfers_count: int + first_level: int + first_time: datetime + # level is not defined in tzkt balances data, so it is + # Level of the block where the token balance was last changed. + last_level: int + last_time: datetime + # owner account + account_address: str | None = None + account_alias: str | None = None + # token object + tzkt_token_id: int | None = None + contract_address: str | None = None + contract_alias: str | None = None + token_id: int | None = None + standard: TzktTokenStandard | None = None + metadata: dict[str, Any] | None = None + + balance: str | None = None + balance_value: float | None = None + + @property + def level(self) -> int: # type: ignore[override] + return self.last_level + + @classmethod + def from_json(cls, token_transfer_json: dict[str, Any]) -> 'TzktTokenBalanceData': + """Convert raw token transfer message from REST or WS into dataclass""" + token_json = token_transfer_json.get('token') or {} + standard = token_json.get('standard') + metadata = token_json.get('metadata') + contract_json = token_json.get('contract') or {} + + return TzktTokenBalanceData( + id=token_transfer_json['id'], + transfers_count=token_transfer_json['transfersCount'], + first_level=token_transfer_json['firstLevel'], + first_time=_parse_timestamp(token_transfer_json['firstTime']), + last_level=token_transfer_json['lastLevel'], + last_time=_parse_timestamp(token_transfer_json['lastTime']), + account_address=token_transfer_json.get('account', {}).get('address'), + account_alias=token_transfer_json.get('account', {}).get('alias'), + tzkt_token_id=token_json['id'], + contract_address=contract_json.get('address'), + contract_alias=contract_json.get('alias'), + token_id=token_json.get('tokenId'), + standard=TzktTokenStandard(standard) if standard else None, + metadata=metadata if isinstance(metadata, dict) else {}, + balance=token_transfer_json.get('balance'), + balance_value=token_transfer_json.get('balanceValue'), + ) + + @dataclass(frozen=True) class TzktEventData(HasLevel): """Basic structure for events received from TzKT REST API""" diff --git a/src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 b/src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 new file mode 100644 index 000000000..757f20925 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 @@ -0,0 +1,21 @@ +spec_version: 2.0 +package: {{ project.package }} + +contracts: + tzbtc_mainnet: + kind: tezos + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: tzbtc + +datasources: + tzkt: + kind: tezos.tzkt + url: https://api.tzkt.io + +indexes: + tzbtc_holders_mainnet: + kind: tezos.tzkt.token_balances + datasource: tzkt + handlers: + - callback: on_balance_update + contract: tzbtc_mainnet diff --git a/src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 b/src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 new file mode 100644 index 000000000..b4a13aee6 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 @@ -0,0 +1,13 @@ +from decimal import Decimal +from {{ project.package }} import models as models +from dipdup.context import HandlerContext +from dipdup.models.tezos_tzkt import TzktTokenBalanceData + + +async def on_balance_update( + ctx: HandlerContext, + token_balance: TzktTokenBalanceData, +) -> None: + holder, _ = await models.Holder.get_or_create(address=token_balance.contract_address) + holder.balance = Decimal(token_balance.balance_value or 0) / (10**8) + await holder.save() \ No newline at end of file diff --git a/src/dipdup/projects/demo_token_balances/models/__init__.py.j2 b/src/dipdup/projects/demo_token_balances/models/__init__.py.j2 new file mode 100644 index 000000000..876bfdd21 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/models/__init__.py.j2 @@ -0,0 +1,8 @@ +from dipdup import fields + +from dipdup.models import Model + + +class Holder(Model): + address = fields.TextField(pk=True) + balance = fields.DecimalField(decimal_places=8, max_digits=20, default=0) diff --git a/src/dipdup/projects/demo_token_balances/replay.yaml b/src/dipdup/projects/demo_token_balances/replay.yaml new file mode 100644 index 000000000..8c8703a05 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/replay.yaml @@ -0,0 +1,5 @@ +spec_version: 2.0 +replay: + description: TzBTC FA1.2 token balances + package: demo_token_balances + template: demo_token_balances \ No newline at end of file diff --git a/tests/configs/demo_token_balances.yml b/tests/configs/demo_token_balances.yml new file mode 100644 index 000000000..3eabede72 --- /dev/null +++ b/tests/configs/demo_token_balances.yml @@ -0,0 +1,23 @@ +spec_version: 2.0 +package: demo_token_balances + +contracts: + tzbtc_mainnet: + kind: tezos + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: tzbtc + +datasources: + tzkt: + kind: tezos.tzkt + url: https://api.tzkt.io + +indexes: + tzbtc_holders_mainnet: + kind: tezos.tzkt.token_balances + datasource: tzkt + first_level: 1366824 + last_level: 1366999 + handlers: + - callback: on_balance_update + contract: tzbtc_mainnet \ No newline at end of file diff --git a/tests/test_demos.py b/tests/test_demos.py index 28f92246a..7e5043f73 100644 --- a/tests/test_demos.py +++ b/tests/test_demos.py @@ -65,6 +65,18 @@ async def assert_run_token_transfers(expected_holders: int, expected_balance: st assert f'{random_balance:f}' == expected_balance +async def assert_run_balances() -> None: + import demo_token_balances.models + + holders = await demo_token_balances.models.Holder.filter().count() + holder = await demo_token_balances.models.Holder.first() + assert holder + random_balance = holder.balance + + assert holders == 1 + assert random_balance == 0 + + async def assert_run_big_maps() -> None: import demo_big_maps.models @@ -171,6 +183,8 @@ async def assert_run_dao() -> None: 'run', partial(assert_run_token_transfers, 2, '-0.02302128'), ), + ('demo_token_balances.yml', 'demo_token_balances', 'run', assert_run_balances), + ('demo_token_balances.yml', 'demo_token_balances', 'init', partial(assert_init, 'demo_token_balances')), ('demo_big_maps.yml', 'demo_big_maps', 'run', assert_run_big_maps), ('demo_big_maps.yml', 'demo_big_maps', 'init', partial(assert_init, 'demo_big_maps')), ('demo_domains.yml', 'demo_domains', 'run', assert_run_domains),