diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb53dbb71..e8eb5def4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,18 +26,18 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: install: true - name: Log in to the registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ env.DOCKER_REGISTRY }} username: ${{ github.actor }} @@ -45,7 +45,7 @@ jobs: - name: Set up metadata id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }} flavor: | @@ -55,7 +55,7 @@ jobs: type=ref,event=tag - name: Publish nightly image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: Dockerfile diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5cf16b51e..d1d824d0b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -35,10 +35,10 @@ jobs: # && sudo apt install gh -y - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@main + uses: actions/setup-python@v4 with: python-version: '3.11' cache: 'pip' @@ -47,7 +47,7 @@ jobs: run: pip install pdm - name: Install project - run: pdm install + run: pdm sync - name: Clone frontend run: | diff --git a/.github/workflows/installer.yml b/.github/workflows/installer.yml index 6d3379640..679a03213 100644 --- a/.github/workflows/installer.yml +++ b/.github/workflows/installer.yml @@ -27,10 +27,10 @@ jobs: arch: arm64 steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@main + uses: actions/setup-python@v4 with: python-version: '3.11' cache: 'pip' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99b3c70f5..e2bbf4011 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,27 +14,27 @@ jobs: steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 # NOTE: Fetch full history for Sentry release with: fetch-depth: 0 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: install: true - name: Log in to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to GHCR - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ env.DOCKER_REGISTRY }} username: ${{ github.actor }} @@ -42,7 +42,7 @@ jobs: - name: Set up Docker metadata id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: images: | dipdup/dipdup @@ -55,7 +55,7 @@ jobs: type=pep440,pattern={{major}}.{{minor}} - name: Set up Python - uses: actions/setup-python@main + uses: actions/setup-python@v4 with: python-version: '3.11' cache: 'pip' @@ -72,7 +72,7 @@ jobs: ALCHEMY_KEY: ${{ secrets.ALCHEMY_KEY }} - name: Publish stable image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: context: . file: Dockerfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 276ee4288..68f01d251 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,19 +28,19 @@ jobs: arch: arm64 steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install PDM run: pipx install pdm - name: Set up Python - uses: actions/setup-python@main + uses: actions/setup-python@v4 with: python-version: '3.11' cache: 'pip' - name: Run install - run: pdm install + run: pdm sync - name: Run lint run: pdm run lint diff --git a/CHANGELOG.md b/CHANGELOG.md index b3409d0f3..a9e25bf27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic ### Added - cli: Added `--unsafe` and `--compose` flags to `config env` command. +- cli: Relative paths to be initialized now can be passed to the `init` command as arguments. +- tezos.tzkt.token_balances: Added new index. + +### Fixed + +- tezos.tzkt.token_transfers: Fixed filtering transfers by token_id. ## [7.0.2] - 2023-10-10 diff --git a/docs/1.getting-started/7.indexes.md b/docs/1.getting-started/7.indexes.md index b023d6582..ddccbe2c7 100644 --- a/docs/1.getting-started/7.indexes.md +++ b/docs/1.getting-started/7.indexes.md @@ -18,6 +18,7 @@ Multiple indexes are available for different workloads. Every index is linked to | [tezos.tzkt.operations](../2.indexes/5.tezos_tzkt_operations.md) | Tezos | TzKT | typed operations | | [tezos.tzkt.operations_unfiltered](../2.indexes/6.tezos_tzkt_operations_unfiltered.md) | Tezos | TzKT | untyped operations | | [tezos.tzkt.token_transfers](../2.indexes/7.tezos_tzkt_token_transfers.md) | Tezos | TzKT | TZIP-12/16 token transfers | +| [tezos.tzkt.token_balances](../2.indexes/8.tezos_tzkt_token_balances.md) | Tezos | TzKT | TZIP-12/16 token balances | Indexes can join multiple contracts considered as a single application. Also, contracts can be used by multiple indexes of any kind, but make sure that they are independent of each other and that indexed data don't overlap. diff --git a/docs/2.indexes/8.tezos_tzkt_token_balances.md b/docs/2.indexes/8.tezos_tzkt_token_balances.md new file mode 100644 index 000000000..12f0ffa1a --- /dev/null +++ b/docs/2.indexes/8.tezos_tzkt_token_balances.md @@ -0,0 +1,19 @@ +--- +title: "Token balances" +description: "This index allows indexing token balances of contracts compatible with FA1.2 or FA2 standards." +network: "tezos" +--- + +# `tezos.tzkt.token_balances` index + +This index allows indexing token balances of contracts compatible with [FA1.2](https://gitlab.com/tzip/tzip/-/blob/master/proposals/tzip-7/README.md) or [FA2](https://gitlab.com/tzip/tzip/-/blob/master/proposals/tzip-12/tzip-12.md) standards. You can either index transfers and cumulatively calculate balances or use this index type to fetch the latest balance information directly. + +```yaml [dipdup.yaml] +{{ #include ../src/demo_token_balances/dipdup.yaml }} +``` + +Callback receives `TzktTokenBalanceData` model that optionally contains the owner, token, and balance values + +```python +{{ #include ../src/demo_token_balances/handlers/on_balance_update.py }} +``` diff --git a/src/demo_token_balances/.dockerignore b/src/demo_token_balances/.dockerignore new file mode 100644 index 000000000..861e17227 --- /dev/null +++ b/src/demo_token_balances/.dockerignore @@ -0,0 +1,22 @@ +# Ignore all +* + +# Add metadata and build files +!demo_token_balances +!pyproject.toml +!pdm.lock +!README.md + +# Add Python code +!**/*.py +**/.*_cache +**/__pycache__ + +# Add configs and scripts (but not env!) +!**/*.graphql +!**/*.json +!**/*.sql +!**/*.yaml +!**/*.yml +!**/*.j2 +!**/.keep \ No newline at end of file diff --git a/src/demo_token_balances/.gitignore b/src/demo_token_balances/.gitignore new file mode 100644 index 000000000..6961da918 --- /dev/null +++ b/src/demo_token_balances/.gitignore @@ -0,0 +1,29 @@ +# Ignore all +* +!*/ + +# Add metadata and build files +!demo_token_balances +!.gitignore +!.dockerignore +!py.typed +!**/Dockerfile +!**/Makefile +!**/pyproject.toml +!**/pdm.lock +!**/README.md +!**/.keep + +# Add Python code +!**/*.py +**/.*_cache +**/__pycache__ + +# Add configs and scripts (but not env!) +!**/*.graphql +!**/*.json +!**/*.sql +!**/*.yaml +!**/*.yml +!**/*.j2 +!**/*.env.default \ No newline at end of file diff --git a/src/demo_token_balances/README.md b/src/demo_token_balances/README.md new file mode 100644 index 000000000..2caca0555 --- /dev/null +++ b/src/demo_token_balances/README.md @@ -0,0 +1,61 @@ +# demo_token_balances + +TzBTC FA1.2 token balances + +## Installation + +This project is based on [DipDup](https://dipdup.io), a framework for building featureful dapps. + +You need a Linux/macOS system with Python 3.11 installed. Use our installer for easy setup: + +```bash +curl -Lsf https://dipdup.io/install.py | python3 +``` + +See the [Installation](https://dipdup.io/docs/installation) page for all options. + +## Usage + +Run the indexer in-memory: + +```bash +dipdup run +``` + +Store data in SQLite database: + +```bash +dipdup -c . -c configs/dipdup.sqlite.yml run +``` + +Or spawn a docker-compose stack: + +```bash +cp deploy/.env.default deploy/.env +# Edit .env before running +docker-compose -f deploy/compose.yaml up +``` + +## Development setup + +We recommend [PDM](https://pdm.fming.dev/latest/) for managing Python projects. To set up the development environment: + +```bash +pdm install +$(pdm venv activate) +``` + +Some tools are included to help you keep the code quality high: black, ruff and mypy. + +```bash +# Format code +pdm fmt + +# Lint code +pdm lint + +# Build Docker image +pdm image +``` + +Inspect the `pyproject.toml` file. It contains all the dependencies and tools used in the project. \ No newline at end of file diff --git a/src/demo_token_balances/__init__.py b/src/demo_token_balances/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/abi/.keep b/src/demo_token_balances/abi/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/configs/.keep b/src/demo_token_balances/configs/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/configs/dipdup.compose.yaml b/src/demo_token_balances/configs/dipdup.compose.yaml new file mode 100644 index 000000000..98824f36f --- /dev/null +++ b/src/demo_token_balances/configs/dipdup.compose.yaml @@ -0,0 +1,22 @@ +database: + kind: postgres + host: ${POSTGRES_HOST:-db} + port: 5432 + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD} + database: ${POSTGRES_DB:-dipdup} + +hasura: + url: http://${HASURA_HOST:-hasura}:8080 + admin_secret: ${HASURA_SECRET} + allow_aggregations: true + camel_case: true + +sentry: + dsn: ${SENTRY_DSN:-""} + environment: ${SENTRY_ENVIRONMENT:-""} + +prometheus: + host: 0.0.0.0 + +logging: ${LOGLEVEL:-INFO} \ No newline at end of file diff --git a/src/demo_token_balances/configs/dipdup.sqlite.yaml b/src/demo_token_balances/configs/dipdup.sqlite.yaml new file mode 100644 index 000000000..ec7006d3d --- /dev/null +++ b/src/demo_token_balances/configs/dipdup.sqlite.yaml @@ -0,0 +1,5 @@ +database: + kind: sqlite + path: ${SQLITE_PATH:-/tmp/demo_token_balances.sqlite} + +logging: ${LOGLEVEL:-INFO} \ No newline at end of file diff --git a/src/demo_token_balances/configs/dipdup.swarm.yaml b/src/demo_token_balances/configs/dipdup.swarm.yaml new file mode 100644 index 000000000..6c7c79b82 --- /dev/null +++ b/src/demo_token_balances/configs/dipdup.swarm.yaml @@ -0,0 +1,22 @@ +database: + kind: postgres + host: ${POSTGRES_HOST:-demo_token_balances_db} + port: 5432 + user: ${POSTGRES_USER:-dipdup} + password: ${POSTGRES_PASSWORD} + database: ${POSTGRES_DB:-dipdup} + +hasura: + url: http://${HASURA_HOST:-demo_token_balances_hasura}:8080 + admin_secret: ${HASURA_SECRET} + allow_aggregations: false + camel_case: true + +sentry: + dsn: ${SENTRY_DSN:-""} + environment: ${SENTRY_ENVIRONMENT:-""} + +prometheus: + host: 0.0.0.0 + +logging: ${LOGLEVEL:-INFO} \ No newline at end of file diff --git a/src/demo_token_balances/configs/replay.yaml b/src/demo_token_balances/configs/replay.yaml new file mode 100644 index 000000000..ef2c6ef31 --- /dev/null +++ b/src/demo_token_balances/configs/replay.yaml @@ -0,0 +1,15 @@ +# Run `dipdup new --replay configs/replay.yaml` to generate new project from this replay +spec_version: 2.0 +replay: + dipdup_version: 7 + template: demo_token_balances + package: demo_token_balances + version: 0.0.1 + description: TzBTC FA1.2 token balances + license: MIT + name: John Doe + email: john_doe@example.com + postgres_image: postgres:15 + postgres_data_path: /var/lib/postgresql/data + hasura_image: hasura/graphql-engine:latest + line_length: 120 diff --git a/src/demo_token_balances/deploy/.env.default b/src/demo_token_balances/deploy/.env.default new file mode 100644 index 000000000..00b262cb5 --- /dev/null +++ b/src/demo_token_balances/deploy/.env.default @@ -0,0 +1,12 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +HASURA_HOST=hasura +HASURA_SECRET= +LOGLEVEL=INFO +POSTGRES_DB=dipdup +POSTGRES_HOST=db +POSTGRES_PASSWORD= +POSTGRES_USER=dipdup +SENTRY_DSN="" +SENTRY_ENVIRONMENT="" diff --git a/src/demo_token_balances/deploy/.keep b/src/demo_token_balances/deploy/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/deploy/Dockerfile b/src/demo_token_balances/deploy/Dockerfile new file mode 100644 index 000000000..b9215c26f --- /dev/null +++ b/src/demo_token_balances/deploy/Dockerfile @@ -0,0 +1,9 @@ +FROM dipdup/dipdup:7 +# FROM ghcr.io/dipdup-io/dipdup:7 +# FROM ghcr.io/dipdup-io/dipdup:next + +# COPY --chown=dipdup pyproject.toml README.md . +# RUN pip install . + +COPY --chown=dipdup . demo_token_balances +WORKDIR demo_token_balances \ No newline at end of file diff --git a/src/demo_token_balances/deploy/compose.sqlite.yaml b/src/demo_token_balances/deploy/compose.sqlite.yaml new file mode 100644 index 000000000..1c7fc50ef --- /dev/null +++ b/src/demo_token_balances/deploy/compose.sqlite.yaml @@ -0,0 +1,19 @@ +version: "3.8" +name: demo_token_balances + +services: + dipdup: + build: + context: .. + dockerfile: deploy/Dockerfile + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.sqlite.yaml", "run"] + restart: always + env_file: .env + ports: + - 46339 + - 9000 + volumes: + - sqlite:${SQLITE_PATH:-/tmp/demo_token_balances.sqlite} + +volumes: + sqlite: \ No newline at end of file diff --git a/src/demo_token_balances/deploy/compose.swarm.yaml b/src/demo_token_balances/deploy/compose.swarm.yaml new file mode 100644 index 000000000..a64536e89 --- /dev/null +++ b/src/demo_token_balances/deploy/compose.swarm.yaml @@ -0,0 +1,92 @@ +version: "3.8" +name: demo_token_balances + +services: + dipdup: + image: ${IMAGE:-ghcr.io/dipdup-io/dipdup}:${TAG:-7} + depends_on: + - db + - hasura + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.swarm.yaml", "run"] + env_file: .env + networks: + - internal + - prometheus-private + deploy: + mode: replicated + replicas: ${INDEXER_ENABLED:-1} + labels: + - prometheus-job=${SERVICE} + - prometheus-port=8000 + placement: &placement + constraints: + - node.labels.${SERVICE} == true + logging: &logging + driver: "json-file" + options: + max-size: "10m" + max-file: "10" + tag: "\{\{.Name\}\}.\{\{.ImageID\}\}" + + db: + image: postgres:15 + volumes: + - db:/var/lib/postgresql/data + env_file: .env + environment: + - POSTGRES_USER=dipdup + - POSTGRES_DB=dipdup + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - internal + deploy: + mode: replicated + replicas: 1 + placement: *placement + logging: *logging + + hasura: + image: hasura/graphql-engine:latest + depends_on: + - db + environment: + - HASURA_GRAPHQL_DATABASE_URL=postgres://dipdup:${POSTGRES_PASSWORD}@demo_token_balances_db:5432/dipdup + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_SECRET} + - HASURA_GRAPHQL_ENABLE_CONSOLE=true + - HASURA_GRAPHQL_DEV_MODE=false + - HASURA_GRAPHQL_LOG_LEVEL=warn + - HASURA_GRAPHQL_ENABLE_TELEMETRY=false + - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user + - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true + networks: + - internal + - traefik-public + deploy: + mode: replicated + replicas: 1 + labels: + - traefik.enable=true + - traefik.http.services.${SERVICE}.loadbalancer.server.port=8080 + - "traefik.http.routers.${SERVICE}.rule=Host(`${HOST}`) && (PathPrefix(`/v1/graphql`) || PathPrefix(`/api/rest`))" + - traefik.http.routers.${SERVICE}.entrypoints=http,${INGRESS:-ingress} + - "traefik.http.routers.${SERVICE}-console.rule=Host(`${SERVICE}.${SWARM_ROOT_DOMAIN}`)" + - traefik.http.routers.${SERVICE}-console.entrypoints=https + - traefik.http.middlewares.${SERVICE}-console.headers.customrequestheaders.X-Hasura-Admin-Secret=${HASURA_SECRET} + - traefik.http.routers.${SERVICE}-console.middlewares=authelia@docker,${SERVICE}-console + placement: *placement + logging: *logging + +volumes: + db: + +networks: + internal: + traefik-public: + external: true + prometheus-private: + external: true \ No newline at end of file diff --git a/src/demo_token_balances/deploy/compose.yaml b/src/demo_token_balances/deploy/compose.yaml new file mode 100644 index 000000000..884b80f27 --- /dev/null +++ b/src/demo_token_balances/deploy/compose.yaml @@ -0,0 +1,55 @@ +version: "3.8" +name: demo_token_balances + +services: + dipdup: + build: + context: .. + dockerfile: deploy/Dockerfile + restart: always + env_file: .env + ports: + - 46339 + - 9000 + command: ["-c", "dipdup.yaml", "-c", "configs/dipdup.compose.yaml", "run"] + depends_on: + - db + - hasura + + db: + image: postgres:15 + ports: + - 5432 + volumes: + - db:/var/lib/postgresql/data + restart: always + env_file: .env + environment: + - POSTGRES_USER=dipdup + - POSTGRES_DB=dipdup + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dipdup"] + interval: 10s + timeout: 5s + retries: 5 + + hasura: + image: hasura/graphql-engine:latest + ports: + - 8080 + depends_on: + - db + restart: always + environment: + - HASURA_GRAPHQL_DATABASE_URL=postgres://dipdup:${POSTGRES_PASSWORD}@db:5432/dipdup + - HASURA_GRAPHQL_ADMIN_SECRET=${HASURA_SECRET} + - HASURA_GRAPHQL_ENABLE_CONSOLE=true + - HASURA_GRAPHQL_DEV_MODE=true + - HASURA_GRAPHQL_LOG_LEVEL=info + - HASURA_GRAPHQL_ENABLE_TELEMETRY=false + - HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user + - HASURA_GRAPHQL_STRINGIFY_NUMERIC_TYPES=true + +volumes: + db: \ No newline at end of file diff --git a/src/demo_token_balances/deploy/sqlite.env.default b/src/demo_token_balances/deploy/sqlite.env.default new file mode 100644 index 000000000..4cacf6bc5 --- /dev/null +++ b/src/demo_token_balances/deploy/sqlite.env.default @@ -0,0 +1,5 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +LOGLEVEL=INFO +SQLITE_PATH=/tmp/demo_token_balances.sqlite diff --git a/src/demo_token_balances/deploy/swarm.env.default b/src/demo_token_balances/deploy/swarm.env.default new file mode 100644 index 000000000..c4811e380 --- /dev/null +++ b/src/demo_token_balances/deploy/swarm.env.default @@ -0,0 +1,12 @@ +# This env file was generated automatically by DipDup. Do not edit it! +# Create a copy with .env extension, fill it with your values and run DipDup with `--env-file` option. +# +HASURA_HOST=demo_token_balances_hasura +HASURA_SECRET= +LOGLEVEL=INFO +POSTGRES_DB=dipdup +POSTGRES_HOST=demo_token_balances_db +POSTGRES_PASSWORD= +POSTGRES_USER=dipdup +SENTRY_DSN="" +SENTRY_ENVIRONMENT="" diff --git a/src/demo_token_balances/dipdup.yaml b/src/demo_token_balances/dipdup.yaml new file mode 100644 index 000000000..da40f0441 --- /dev/null +++ b/src/demo_token_balances/dipdup.yaml @@ -0,0 +1,21 @@ +spec_version: 2.0 +package: demo_token_balances + +contracts: + tzbtc_mainnet: + kind: tezos + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: tzbtc + +datasources: + tzkt: + kind: tezos.tzkt + url: https://api.tzkt.io + +indexes: + tzbtc_holders_mainnet: + kind: tezos.tzkt.token_balances + datasource: tzkt + handlers: + - callback: on_balance_update + contract: tzbtc_mainnet \ No newline at end of file diff --git a/src/demo_token_balances/graphql/.keep b/src/demo_token_balances/graphql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/handlers/.keep b/src/demo_token_balances/handlers/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/handlers/on_balance_update.py b/src/demo_token_balances/handlers/on_balance_update.py new file mode 100644 index 000000000..ec6716d4d --- /dev/null +++ b/src/demo_token_balances/handlers/on_balance_update.py @@ -0,0 +1,14 @@ +from decimal import Decimal + +from demo_token_balances import models as models +from dipdup.context import HandlerContext +from dipdup.models.tezos_tzkt import TzktTokenBalanceData + + +async def on_balance_update( + ctx: HandlerContext, + token_balance: TzktTokenBalanceData, +) -> None: + holder, _ = await models.Holder.get_or_create(address=token_balance.contract_address) + holder.balance = Decimal(token_balance.balance_value or 0) / (10**8) + await holder.save() \ No newline at end of file diff --git a/src/demo_token_balances/hasura/.keep b/src/demo_token_balances/hasura/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/hooks/.keep b/src/demo_token_balances/hooks/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/hooks/on_index_rollback.py b/src/demo_token_balances/hooks/on_index_rollback.py new file mode 100644 index 000000000..3d38655e0 --- /dev/null +++ b/src/demo_token_balances/hooks/on_index_rollback.py @@ -0,0 +1,16 @@ +from dipdup.context import HookContext +from dipdup.index import Index + + +async def on_index_rollback( + ctx: HookContext, + index: Index, # type: ignore[type-arg] + from_level: int, + to_level: int, +) -> None: + await ctx.execute_sql('on_index_rollback') + await ctx.rollback( + index=index.name, + from_level=from_level, + to_level=to_level, + ) \ No newline at end of file diff --git a/src/demo_token_balances/hooks/on_reindex.py b/src/demo_token_balances/hooks/on_reindex.py new file mode 100644 index 000000000..0804aae37 --- /dev/null +++ b/src/demo_token_balances/hooks/on_reindex.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_reindex( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_reindex') \ No newline at end of file diff --git a/src/demo_token_balances/hooks/on_restart.py b/src/demo_token_balances/hooks/on_restart.py new file mode 100644 index 000000000..2581b5be3 --- /dev/null +++ b/src/demo_token_balances/hooks/on_restart.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_restart( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_restart') \ No newline at end of file diff --git a/src/demo_token_balances/hooks/on_synchronized.py b/src/demo_token_balances/hooks/on_synchronized.py new file mode 100644 index 000000000..09099e4b6 --- /dev/null +++ b/src/demo_token_balances/hooks/on_synchronized.py @@ -0,0 +1,7 @@ +from dipdup.context import HookContext + + +async def on_synchronized( + ctx: HookContext, +) -> None: + await ctx.execute_sql('on_synchronized') \ No newline at end of file diff --git a/src/demo_token_balances/models/.keep b/src/demo_token_balances/models/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/models/__init__.py b/src/demo_token_balances/models/__init__.py new file mode 100644 index 000000000..833a8deea --- /dev/null +++ b/src/demo_token_balances/models/__init__.py @@ -0,0 +1,7 @@ +from dipdup import fields +from dipdup.models import Model + + +class Holder(Model): + address = fields.TextField(pk=True) + balance = fields.DecimalField(decimal_places=8, max_digits=20, default=0) \ No newline at end of file diff --git a/src/demo_token_balances/py.typed b/src/demo_token_balances/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/pyproject.toml b/src/demo_token_balances/pyproject.toml new file mode 100644 index 000000000..32ad2b1ff --- /dev/null +++ b/src/demo_token_balances/pyproject.toml @@ -0,0 +1,52 @@ +[project] +name = "demo_token_balances" +version = "0.0.1" +description = "TzBTC FA1.2 token balances" +license = { text = "MIT" } +authors = [ + { name = "John Doe", email = "john_doe@example.com" } +] +readme = "README.md" +requires-python = ">=3.11,<3.12" +dependencies = [ + "dipdup>=7,<8" +] + +[tool.pdm.dev-dependencies] +dev = [ + "isort", + "black", + "ruff", + "mypy", +] + +[tool.pdm.scripts] +_isort = "isort ." +_black = "black ." +_ruff = "ruff check --fix ." +_mypy = "mypy --no-incremental --exclude demo_token_balances ." +all = { composite = ["fmt", "lint"] } +fmt = { composite = ["_isort", "_black"] } +lint = { composite = ["_ruff", "_mypy"] } +image = "docker buildx build . --load --progress plain -f deploy/Dockerfile -t demo_token_balances:latest" + +[tool.isort] +line_length = 120 +force_single_line = true + +[tool.black] +line-length = 120 +target-version = ['py311'] +skip-string-normalization = true + +[tool.ruff] +line-length = 120 +target-version = 'py311' + +[tool.mypy] +python_version = "3.11" +plugins = ["pydantic.mypy"] + +[build-system] +requires = ["pdm-backend"] +build-backend = "pdm.backend" \ No newline at end of file diff --git a/src/demo_token_balances/sql/.keep b/src/demo_token_balances/sql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_index_rollback/.keep b/src/demo_token_balances/sql/on_index_rollback/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_reindex/.keep b/src/demo_token_balances/sql/on_reindex/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_restart/.keep b/src/demo_token_balances/sql/on_restart/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/sql/on_synchronized/.keep b/src/demo_token_balances/sql/on_synchronized/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_token_balances/types/.keep b/src/demo_token_balances/types/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 11fcb6a56..bb35be168 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -288,9 +288,20 @@ async def run(ctx: click.Context) -> None: @cli.command() @click.option('--force', '-f', is_flag=True, help='Overwrite existing types and ABIs.') @click.option('--base', '-b', is_flag=True, help='Include template base: pyproject.toml, Dockerfile, etc.') +@click.argument( + 'include', + type=str, + nargs=-1, + metavar='PATH', +) @click.pass_context @_cli_wrapper -async def init(ctx: click.Context, force: bool, base: bool) -> None: +async def init( + ctx: click.Context, + force: bool, + base: bool, + include: list[str], +) -> None: """Generate project tree, typeclasses and callback stubs. This command is idempotent, meaning it won't overwrite previously generated files unless asked explicitly. @@ -299,7 +310,12 @@ async def init(ctx: click.Context, force: bool, base: bool) -> None: config: DipDupConfig = ctx.obj.config dipdup = DipDup(config) - await dipdup.init(force, base) + + await dipdup.init( + force=force, + base=base or bool(include), + include=set(include), + ) @cli.command() @@ -517,6 +533,7 @@ async def schema_wipe(ctx: click.Context, immune: bool, force: bool) -> None: models=models, timeout=config.database.connection_timeout, decimal_precision=config.advanced.decimal_precision, + unsafe_sqlite=config.advanced.unsafe_sqlite, ): conn = get_connection() await wipe_schema( diff --git a/src/dipdup/codegen/__init__.py b/src/dipdup/codegen/__init__.py index ce2de0b97..933124a3b 100644 --- a/src/dipdup/codegen/__init__.py +++ b/src/dipdup/codegen/__init__.py @@ -38,10 +38,12 @@ def __init__( config: DipDupConfig, package: DipDupPackage, datasources: dict[str, Datasource[Any]], + include: set[str] | None = None, ) -> None: self._config = config self._package = package self._datasources = datasources + self._include = include or set() self._logger = _logger async def init( @@ -52,9 +54,14 @@ async def init( self._package.create() replay = self._package.replay - if base and replay: + if base: + if not replay: + raise FrameworkException('`--base` option passed but `configs/replay.yaml` file is missing') _logger.info('Recreating base template with replay.yaml') - render_base(replay, force) + render_base(replay, force, self._include) + + if self._include: + force = any(str(path).startswith('types') for path in self._include) await self.generate_abi() await self.generate_schemas(force) diff --git a/src/dipdup/codegen/tezos_tzkt.py b/src/dipdup/codegen/tezos_tzkt.py index 0b4a0082a..51ff82e54 100644 --- a/src/dipdup/codegen/tezos_tzkt.py +++ b/src/dipdup/codegen/tezos_tzkt.py @@ -90,11 +90,13 @@ def __init__( config: DipDupConfig, package: DipDupPackage, datasources: dict[str, Datasource[Any]], + include: set[str] | None = None, ) -> None: super().__init__( config=config, package=package, datasources=datasources, + include=include, ) self._schemas: dict[str, dict[str, dict[str, Any]]] = {} diff --git a/src/dipdup/config/__init__.py b/src/dipdup/config/__init__.py index 6ec3461a3..8740486aa 100644 --- a/src/dipdup/config/__init__.py +++ b/src/dipdup/config/__init__.py @@ -956,13 +956,20 @@ def _resolve_index_links(self, index_config: ResolvedIndexConfigU) -> None: handler_config.parent = index_config if isinstance(handler_config.contract, str): - handler_config.contract = self.get_contract(handler_config.contract) + handler_config.contract = self.get_tezos_contract(handler_config.contract) if isinstance(handler_config.from_, str): - handler_config.from_ = self.get_contract(handler_config.from_) + handler_config.from_ = self.get_tezos_contract(handler_config.from_) if isinstance(handler_config.to, str): - handler_config.to = self.get_contract(handler_config.to) + handler_config.to = self.get_tezos_contract(handler_config.to) + + elif isinstance(index_config, TzktTokenBalancesIndexConfig): + for handler_config in index_config.handlers: + handler_config.parent = index_config + + if isinstance(handler_config.contract, str): + handler_config.contract = self.get_tezos_contract(handler_config.contract) elif isinstance(index_config, TzktOperationsUnfilteredIndexConfig): index_config.handler_config.parent = index_config @@ -1025,6 +1032,7 @@ def _set_names(self) -> None: from dipdup.config.tezos_tzkt_operations import OperationsHandlerTransactionPatternConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsIndexConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsUnfilteredIndexConfig +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesIndexConfig from dipdup.config.tezos_tzkt_token_transfers import TzktTokenTransfersIndexConfig from dipdup.config.tzip_metadata import TzipMetadataDatasourceConfig @@ -1048,6 +1056,7 @@ def _set_names(self) -> None: | TzktOperationsIndexConfig | TzktOperationsUnfilteredIndexConfig | TzktTokenTransfersIndexConfig + | TzktTokenBalancesIndexConfig ) IndexConfigU = ResolvedIndexConfigU | IndexTemplateConfig diff --git a/src/dipdup/config/tezos_tzkt_token_balances.py b/src/dipdup/config/tezos_tzkt_token_balances.py new file mode 100644 index 000000000..3bb3fb754 --- /dev/null +++ b/src/dipdup/config/tezos_tzkt_token_balances.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Literal + +from pydantic.dataclasses import dataclass +from pydantic.fields import Field + +from dipdup.config import ContractConfig +from dipdup.config import HandlerConfig +from dipdup.config.tezos import TezosContractConfig +from dipdup.config.tezos_tzkt import TzktDatasourceConfig +from dipdup.config.tezos_tzkt import TzktIndexConfig +from dipdup.models.tezos_tzkt import TokenBalanceSubscription + +if TYPE_CHECKING: + from collections.abc import Iterator + + from dipdup.subscriptions import Subscription + + +@dataclass +class TzktTokenBalancesHandlerConfig(HandlerConfig): + """Token balance handler config + + :param callback: Callback name + :param contract: Filter by contract + :param token_id: Filter by token ID + """ + + contract: TezosContractConfig | None = None + token_id: int | None = None + + def iter_imports(self, package: str) -> Iterator[tuple[str, str]]: + """This iterator result will be used in codegen to generate handler(s) template""" + yield 'dipdup.context', 'HandlerContext' + yield 'dipdup.models.tezos_tzkt', 'TzktTokenBalanceData' + yield package, 'models as models' + + def iter_arguments(self) -> Iterator[tuple[str, str]]: + """This iterator result will be used in codegen to generate handler(s) template""" + yield 'ctx', 'HandlerContext' + yield 'token_balance', 'TzktTokenBalanceData' + + +@dataclass +class TzktTokenBalancesIndexConfig(TzktIndexConfig): + """Token balance index config + + :param kind: always `tezos.tzkt.token_balances` + :param datasource: Index datasource to use + :param handlers: Mapping of token transfer handlers + + :param first_level: Level to start indexing from + :param last_level: Level to stop indexing at + """ + + kind: Literal['tezos.tzkt.token_balances'] + datasource: TzktDatasourceConfig + handlers: tuple[TzktTokenBalancesHandlerConfig, ...] = Field(default_factory=tuple) + + first_level: int = 0 + last_level: int = 0 + + def get_subscriptions(self) -> set[Subscription]: + subs = super().get_subscriptions() + if self.datasource.merge_subscriptions: + subs.add(TokenBalanceSubscription()) + else: + for handler_config in self.handlers: + contract = ( + handler_config.contract.address if isinstance(handler_config.contract, ContractConfig) else None + ) + subs.add( + TokenBalanceSubscription( + contract=contract, + token_id=handler_config.token_id, + ) + ) + return subs diff --git a/src/dipdup/context.py b/src/dipdup/context.py index 1407e6116..32e24a810 100644 --- a/src/dipdup/context.py +++ b/src/dipdup/context.py @@ -33,6 +33,7 @@ from dipdup.config.tezos_tzkt_head import TzktHeadIndexConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsIndexConfig from dipdup.config.tezos_tzkt_operations import TzktOperationsUnfilteredIndexConfig +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesIndexConfig from dipdup.config.tezos_tzkt_token_transfers import TzktTokenTransfersIndexConfig from dipdup.database import execute_sql from dipdup.database import execute_sql_query @@ -297,10 +298,11 @@ async def _spawn_index(self, name: str, state: Index | None = None) -> Any: from dipdup.indexes.tezos_tzkt_events.index import TzktEventsIndex from dipdup.indexes.tezos_tzkt_head.index import TzktHeadIndex from dipdup.indexes.tezos_tzkt_operations.index import TzktOperationsIndex + from dipdup.indexes.tezos_tzkt_token_balances.index import TzktTokenBalancesIndex from dipdup.indexes.tezos_tzkt_token_transfers.index import TzktTokenTransfersIndex index_config = cast(ResolvedIndexConfigU, self.config.get_index(name)) - index: TzktOperationsIndex | TzktBigMapsIndex | TzktHeadIndex | TzktTokenTransfersIndex | TzktEventsIndex | SubsquidEventsIndex + index: TzktOperationsIndex | TzktBigMapsIndex | TzktHeadIndex | TzktTokenBalancesIndex | TzktTokenTransfersIndex | TzktEventsIndex | SubsquidEventsIndex datasource_name = index_config.datasource.name datasource: TzktDatasource | SubsquidDatasource @@ -315,6 +317,9 @@ async def _spawn_index(self, name: str, state: Index | None = None) -> Any: elif isinstance(index_config, TzktHeadIndexConfig): datasource = self.get_tzkt_datasource(datasource_name) index = TzktHeadIndex(self, index_config, datasource) + elif isinstance(index_config, TzktTokenBalancesIndexConfig): + datasource = self.get_tzkt_datasource(datasource_name) + index = TzktTokenBalancesIndex(self, index_config, datasource) elif isinstance(index_config, TzktTokenTransfersIndexConfig): datasource = self.get_tzkt_datasource(datasource_name) index = TzktTokenTransfersIndex(self, index_config, datasource) diff --git a/src/dipdup/database.py b/src/dipdup/database.py index cff3cf138..da66b20f3 100644 --- a/src/dipdup/database.py +++ b/src/dipdup/database.py @@ -214,12 +214,40 @@ async def _pg_create_functions(conn: AsyncpgClient) -> None: await execute_sql(conn, sql_path) +async def get_tables() -> set[str]: + conn = get_connection() + if isinstance(conn, SqliteClient): + _, sqlite_res = await conn.execute_query('SELECT name FROM sqlite_master WHERE type = "table";') + return {row[0] for row in sqlite_res} + if isinstance(conn, AsyncpgClient): + _, postgres_res = await conn.execute_query( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE'" + ) + return {row[0] for row in postgres_res} + + raise NotImplementedError + + async def _pg_create_views(conn: AsyncpgClient) -> None: sql_path = Path(__file__).parent / 'sql' / 'dipdup_head_status.sql' # TODO: Configurable interval await execute_sql(conn, sql_path, HEAD_STATUS_TIMEOUT) +# FIXME: Private but used in dipdup.hasura +async def _pg_get_views(conn: AsyncpgClient, schema_name: str) -> list[str]: + return [ + row[0] + for row in ( + await conn.execute_query( + "SELECT table_name FROM information_schema.views WHERE table_schema =" + f" '{schema_name}' UNION SELECT matviewname as table_name FROM pg_matviews" + f" WHERE schemaname = '{schema_name}'" + ) + )[1] + ] + + async def _pg_wipe_schema( conn: AsyncpgClient, schema_name: str, @@ -257,8 +285,11 @@ async def _sqlite_wipe_schema( # NOTE: Copy immune tables to the new database. master_query = 'SELECT name FROM sqlite_master WHERE type = "table"' result = await conn.execute_query(master_query) - for name in result[1]: - if name not in immune_tables: # type: ignore[comparison-overlap] + for row in result[1]: + name = row[0] + if name == 'sqlite_sequence': + continue + if name not in immune_tables: continue expr = f'CREATE TABLE {namespace}.{name} AS SELECT * FROM {name}' diff --git a/src/dipdup/datasources/tezos_tzkt.py b/src/dipdup/datasources/tezos_tzkt.py index 00fd4d479..72bbebcf3 100644 --- a/src/dipdup/datasources/tezos_tzkt.py +++ b/src/dipdup/datasources/tezos_tzkt.py @@ -43,6 +43,7 @@ from dipdup.models.tezos_tzkt import TzktQuoteData from dipdup.models.tezos_tzkt import TzktRollbackMessage from dipdup.models.tezos_tzkt import TzktSubscription +from dipdup.models.tezos_tzkt import TzktTokenBalanceData from dipdup.models.tezos_tzkt import TzktTokenTransferData from dipdup.utils import split_by_chunks @@ -111,6 +112,18 @@ 'originationId', 'migrationId', ) +TOKEN_BALANCE_FIELDS = ( + 'id', + 'transfersCount', + 'firstLevel', + 'firstTime', + 'lastLevel', + 'lastTime', + 'account', + 'token', + 'balance', + 'balanceValue', +) EVENT_FIELDS = ( 'id', 'level', @@ -127,6 +140,7 @@ HeadCallback = Callable[['TzktDatasource', TzktHeadBlockData], Awaitable[None]] OperationsCallback = Callable[['TzktDatasource', tuple[TzktOperationData, ...]], Awaitable[None]] TokenTransfersCallback = Callable[['TzktDatasource', tuple[TzktTokenTransferData, ...]], Awaitable[None]] +TokenBalancesCallback = Callable[['TzktDatasource', tuple[TzktTokenBalanceData, ...]], Awaitable[None]] BigMapsCallback = Callable[['TzktDatasource', tuple[TzktBigMapData, ...]], Awaitable[None]] EventsCallback = Callable[['TzktDatasource', tuple[TzktEventData, ...]], Awaitable[None]] RollbackCallback = Callable[['TzktDatasource', MessageType, int, int], Awaitable[None]] @@ -236,6 +250,7 @@ def __init__( self._on_head_callbacks: set[HeadCallback] = set() self._on_operations_callbacks: set[OperationsCallback] = set() self._on_token_transfers_callbacks: set[TokenTransfersCallback] = set() + self._on_token_balances_callbacks: set[TokenBalancesCallback] = set() self._on_big_maps_callbacks: set[BigMapsCallback] = set() self._on_events_callbacks: set[EventsCallback] = set() self._on_rollback_callbacks: set[RollbackCallback] = set() @@ -341,6 +356,10 @@ async def emit_token_transfers(self, token_transfers: tuple[TzktTokenTransferDat for fn in self._on_token_transfers_callbacks: await fn(self, token_transfers) + async def emit_token_balances(self, token_balances: tuple[TzktTokenBalanceData, ...]) -> None: + for fn in self._on_token_balances_callbacks: + await fn(self, token_balances) + async def emit_big_maps(self, big_maps: tuple[TzktBigMapData, ...]) -> None: for fn in self._on_big_maps_callbacks: await fn(self, big_maps) @@ -897,6 +916,48 @@ async def iter_token_transfers( ): yield batch + async def get_token_balances( + self, + token_addresses: set[str], + token_ids: set[int], + first_level: int | None = None, + last_level: int | None = None, + offset: int | None = None, + limit: int | None = None, + ) -> tuple[TzktTokenBalanceData, ...]: + params = self._get_request_params( + first_level, + last_level, + offset=offset or 0, + limit=limit, + select=TOKEN_BALANCE_FIELDS, + values=True, + cursor=True, + **{ + 'token.contract.in': ','.join(token_addresses), + 'token.id.in': ','.join(str(token_id) for token_id in token_ids), + }, + ) + raw_token_balances = await self._request_values_dict('get', url='v1/tokens/balances', params=params) + return tuple(TzktTokenBalanceData.from_json(item) for item in raw_token_balances) + + async def iter_token_balances( + self, + token_addresses: set[str], + token_ids: set[int], + first_level: int | None = None, + last_level: int | None = None, + ) -> AsyncIterator[tuple[TzktTokenBalanceData, ...]]: + async for batch in self._iter_batches( + self.get_token_balances, + token_addresses, + token_ids, + first_level, + last_level, + cursor=True, + ): + yield batch + async def get_events( self, addresses: set[str], @@ -1073,6 +1134,7 @@ def _get_signalr_client(self) -> SignalRClient: self._signalr_client.on('operations', partial(self._on_message, TzktMessageType.operation)) self._signalr_client.on('transfers', partial(self._on_message, TzktMessageType.token_transfer)) + self._signalr_client.on('balances', partial(self._on_message, TzktMessageType.token_balance)) self._signalr_client.on('bigmaps', partial(self._on_message, TzktMessageType.big_map)) self._signalr_client.on('head', partial(self._on_message, TzktMessageType.head)) self._signalr_client.on('events', partial(self._on_message, TzktMessageType.event)) @@ -1146,6 +1208,8 @@ async def _on_message(self, type_: TzktMessageType, message: list[dict[str, Any] await self._process_operations_data(cast(list[dict[str, Any]], buffered_message.data)) elif buffered_message.type == TzktMessageType.token_transfer: await self._process_token_transfers_data(cast(list[dict[str, Any]], buffered_message.data)) + elif buffered_message.type == TzktMessageType.token_balance: + await self._process_token_balances_data(cast(list[dict[str, Any]], buffered_message.data)) elif buffered_message.type == TzktMessageType.big_map: await self._process_big_maps_data(cast(list[dict[str, Any]], buffered_message.data)) elif buffered_message.type == TzktMessageType.head: @@ -1182,6 +1246,17 @@ async def _process_token_transfers_data(self, data: list[dict[str, Any]]) -> Non for _level, token_transfers in level_token_transfers.items(): await self.emit_token_transfers(tuple(token_transfers)) + async def _process_token_balances_data(self, data: list[dict[str, Any]]) -> None: + """Parse and emit raw token balances from WS""" + level_token_balances: defaultdict[int, deque[TzktTokenBalanceData]] = defaultdict(deque) + + for token_balance_json in data: + token_balance = TzktTokenBalanceData.from_json(token_balance_json) + level_token_balances[token_balance.level].append(token_balance) + + for _level, token_balances in level_token_balances.items(): + await self.emit_token_balances(tuple(token_balances)) + async def _process_big_maps_data(self, data: list[dict[str, Any]]) -> None: """Parse and emit raw big map diffs from WS""" level_big_maps: defaultdict[int, deque[TzktBigMapData]] = defaultdict(deque) diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index 258f3b1b3..acff12dbd 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -20,6 +20,7 @@ from tortoise.exceptions import OperationalError from dipdup import env +from dipdup.codegen import CodeGenerator from dipdup.codegen import generate_environments from dipdup.config import DipDupConfig from dipdup.config import IndexTemplateConfig @@ -502,6 +503,7 @@ async def init( self, force: bool = False, base: bool = False, + include: set[str] | None = None, ) -> None: """Create new or update existing dipdup project""" from dipdup.codegen.evm_subsquid import SubsquidCodeGenerator @@ -515,8 +517,17 @@ async def init( package = DipDupPackage(self._config.package_path) - for codegen_cls in (TzktCodeGenerator, SubsquidCodeGenerator): - codegen = codegen_cls(self._config, package, self._datasources) + codegen_classes: tuple[type[CodeGenerator], ...] = ( + TzktCodeGenerator, + SubsquidCodeGenerator, + ) + for codegen_cls in codegen_classes: + codegen = codegen_cls( + config=self._config, + package=package, + datasources=self._datasources, + include=include, + ) await codegen.init( force=force, base=base, diff --git a/src/dipdup/hasura.py b/src/dipdup/hasura.py index 2fe04b260..f54885dde 100644 --- a/src/dipdup/hasura.py +++ b/src/dipdup/hasura.py @@ -19,6 +19,8 @@ from dipdup.config import HttpConfig from dipdup.config import PostgresDatabaseConfig from dipdup.config import ResolvedHttpConfig +from dipdup.database import AsyncpgClient +from dipdup.database import _pg_get_views from dipdup.database import get_connection from dipdup.database import iter_models from dipdup.exceptions import ConfigurationError @@ -328,18 +330,9 @@ async def _apply_custom_metadata(self) -> None: async def _get_views(self) -> list[str]: conn = get_connection() - views = [ - row[0] - for row in ( - await conn.execute_query( - "SELECT table_name FROM information_schema.views WHERE table_schema =" - f" '{self._database_config.schema_name}' UNION SELECT matviewname as table_name FROM pg_matviews" - f" WHERE schemaname = '{self._database_config.schema_name}'" - ) - )[1] - ] - self._logger.info('Found %s regular and materialized views', len(views)) - return views + if not isinstance(conn, AsyncpgClient): + raise HasuraError('Hasura integration requires `postgres` database client') + return await _pg_get_views(conn, self._database_config.schema_name) def _iterate_graphql_queries(self) -> Iterator[tuple[str, str]]: graphql_path = env.get_package_path(self._package) / 'graphql' diff --git a/src/dipdup/indexes/tezos_tzkt_token_balances/__init__.py b/src/dipdup/indexes/tezos_tzkt_token_balances/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/dipdup/indexes/tezos_tzkt_token_balances/index.py b/src/dipdup/indexes/tezos_tzkt_token_balances/index.py new file mode 100644 index 000000000..710adf028 --- /dev/null +++ b/src/dipdup/indexes/tezos_tzkt_token_balances/index.py @@ -0,0 +1,114 @@ +from contextlib import ExitStack + +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesHandlerConfig +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesIndexConfig +from dipdup.datasources.tezos_tzkt import TzktDatasource +from dipdup.exceptions import ConfigInitializationException +from dipdup.exceptions import FrameworkException +from dipdup.index import Index +from dipdup.indexes.tezos_tzkt_token_balances.matcher import match_token_balances +from dipdup.models.tezos_tzkt import TzktMessageType +from dipdup.models.tezos_tzkt import TzktRollbackMessage +from dipdup.models.tezos_tzkt import TzktTokenBalanceData +from dipdup.prometheus import Metrics + +TokenBalanceQueueItem = tuple[TzktTokenBalanceData, ...] | TzktRollbackMessage + + +class TzktTokenBalancesIndex( + Index[TzktTokenBalancesIndexConfig, TokenBalanceQueueItem, TzktDatasource], + message_type=TzktMessageType.token_balance, +): + def push_token_balances(self, token_balances: TokenBalanceQueueItem) -> None: + self.push_realtime_message(token_balances) + + async def _synchronize(self, sync_level: int) -> None: + await self._enter_sync_state(sync_level) + await self._synchronize_actual(sync_level) + await self._exit_sync_state(sync_level) + + async def _synchronize_actual(self, head_level: int) -> None: + """Retrieve data for the current level""" + # TODO: think about logging and metrics + + addresses, token_ids = set(), set() + for handler in self._config.handlers: + if handler.contract and handler.contract.address is not None: + addresses.add(handler.contract.address) + if handler.token_id is not None: + token_ids.add(handler.token_id) + + async with self._ctx.transactions.in_transaction(head_level, head_level, self.name): + # NOTE: If index is out of date fetch balances as of the current head. + async for balances_batch in self._datasource.iter_token_balances( + addresses, token_ids, last_level=head_level + ): + matched_handlers = match_token_balances(self._config.handlers, balances_batch) + for handler_config, matched_balance_data in matched_handlers: + await self._call_matched_handler(handler_config, matched_balance_data) + + await self._update_state(level=head_level) + + async def _process_level_token_balances( + self, + token_balances: tuple[TzktTokenBalanceData, ...], + sync_level: int, + ) -> None: + if not token_balances: + return + + batch_level = token_balances[0].level + index_level = self.state.level + if batch_level <= index_level: + raise FrameworkException(f'Batch level is lower than index level: {batch_level} <= {index_level}') + + self._logger.debug('Processing token balances of level %s', batch_level) + matched_handlers = match_token_balances(self._config.handlers, token_balances) + + if Metrics.enabled: + Metrics.set_index_handlers_matched(len(matched_handlers)) + + # NOTE: We still need to bump index level but don't care if it will be done in existing transaction + if not matched_handlers: + await self._update_state(level=batch_level) + return + + async with self._ctx.transactions.in_transaction(batch_level, sync_level, self.name): + for handler_config, token_balance in matched_handlers: + await self._call_matched_handler(handler_config, token_balance) + await self._update_state(level=batch_level) + + async def _call_matched_handler( + self, handler_config: TzktTokenBalancesHandlerConfig, token_balance: TzktTokenBalanceData + ) -> None: + if not handler_config.parent: + raise ConfigInitializationException + + await self._ctx.fire_handler( + handler_config.callback, + handler_config.parent.name, + self.datasource, + # NOTE: missing `operation_id` field in API to identify operation + None, + token_balance, + ) + + async def _process_queue(self) -> None: + """Process WebSocket queue""" + if self._queue: + self._logger.debug('Processing websocket queue') + while self._queue: + message = self._queue.popleft() + if isinstance(message, TzktRollbackMessage): + await self._tzkt_rollback(message.from_level, message.to_level) + continue + + message_level = message[0].level + if message_level <= self.state.level: + self._logger.debug('Skipping outdated message: %s <= %s', message_level, self.state.level) + continue + + with ExitStack() as stack: + if Metrics.enabled: + stack.enter_context(Metrics.measure_level_realtime_duration()) + await self._process_level_token_balances(message, message_level) diff --git a/src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py b/src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py new file mode 100644 index 000000000..46b00a015 --- /dev/null +++ b/src/dipdup/indexes/tezos_tzkt_token_balances/matcher.py @@ -0,0 +1,42 @@ +import logging +from collections import deque +from collections.abc import Iterable + +from dipdup.config.tezos_tzkt_token_balances import TzktTokenBalancesHandlerConfig +from dipdup.models.tezos_tzkt import TzktTokenBalanceData + +_logger = logging.getLogger('dipdup.matcher') + +MatchedTokenBalancesT = tuple[TzktTokenBalancesHandlerConfig, TzktTokenBalanceData] + + +def match_token_balance( + handler_config: TzktTokenBalancesHandlerConfig, + token_balance: TzktTokenBalanceData, +) -> bool: + """Match single token balance with pattern""" + if handler_config.contract: + if handler_config.contract.address != token_balance.contract_address: + return False + if handler_config.token_id is not None: + if handler_config.token_id != token_balance.token_id: + return False + return True + + +def match_token_balances( + handlers: Iterable[TzktTokenBalancesHandlerConfig], token_balances: Iterable[TzktTokenBalanceData] +) -> deque[MatchedTokenBalancesT]: + """Try to match token balances with all index handlers.""" + + matched_handlers: deque[MatchedTokenBalancesT] = deque() + + for token_balance in token_balances: + for handler_config in handlers: + token_balance_matched = match_token_balance(handler_config, token_balance) + if not token_balance_matched: + continue + _logger.debug('%s: `%s` handler matched!', token_balance.level, handler_config.callback) + matched_handlers.append((handler_config, token_balance)) + + return matched_handlers diff --git a/src/dipdup/models/__init__.py b/src/dipdup/models/__init__.py index 85997fd3f..519188a48 100644 --- a/src/dipdup/models/__init__.py +++ b/src/dipdup/models/__init__.py @@ -53,6 +53,7 @@ class IndexType(Enum): tezos_tzkt_big_maps = 'tezos.tzkt.big_maps' tezos_tzkt_head = 'tezos.tzkt.head' tezos_tzkt_token_transfers = 'tezos.tzkt.token_transfers' + tezos_tzkt_token_balances = 'tezos.tzkt.token_balances' tezos_tzkt_events = 'tezos.tzkt.events' evm_subsquid_events = 'evm.subsquid.events' diff --git a/src/dipdup/models/tezos_tzkt.py b/src/dipdup/models/tezos_tzkt.py index dd9085180..b63a9679a 100644 --- a/src/dipdup/models/tezos_tzkt.py +++ b/src/dipdup/models/tezos_tzkt.py @@ -48,6 +48,7 @@ class TzktMessageType(MessageType, Enum): big_map = 'big_map' head = 'head' token_transfer = 'token_transfer' + token_balance = 'token_balance' event = 'event' @@ -119,7 +120,7 @@ class TokenTransferSubscription(TzktSubscription): def get_request(self) -> list[dict[str, Any]]: request: dict[str, Any] = {} if self.token_id: - request['token_id'] = self.token_id + request['tokenId'] = self.token_id if self.contract: request['contract'] = self.contract if self.from_: @@ -129,6 +130,22 @@ def get_request(self) -> list[dict[str, Any]]: return [request] +@dataclass(frozen=True) +class TokenBalanceSubscription(TzktSubscription): + type: Literal['token_balance'] = 'token_balance' + method: Literal['SubscribeToTokenBalances'] = 'SubscribeToTokenBalances' + contract: str | None = None + token_id: int | None = None + + def get_request(self) -> list[dict[str, Any]]: + request: dict[str, Any] = {} + if self.token_id: + request['tokenId'] = self.token_id + if self.contract: + request['contract'] = self.contract + return [request] + + @dataclass(frozen=True) class EventSubscription(TzktSubscription): type: Literal['event'] = 'event' @@ -537,6 +554,64 @@ def from_json(cls, token_transfer_json: dict[str, Any]) -> 'TzktTokenTransferDat ) +@dataclass(frozen=True) +class TzktTokenBalanceData(HasLevel): + """Basic structure for token transver received from TzKT SignalR API""" + + id: int + transfers_count: int + first_level: int + first_time: datetime + # level is not defined in tzkt balances data, so it is + # Level of the block where the token balance was last changed. + last_level: int + last_time: datetime + # owner account + account_address: str | None = None + account_alias: str | None = None + # token object + tzkt_token_id: int | None = None + contract_address: str | None = None + contract_alias: str | None = None + token_id: int | None = None + standard: TzktTokenStandard | None = None + metadata: dict[str, Any] | None = None + + balance: str | None = None + balance_value: float | None = None + + @property + def level(self) -> int: # type: ignore[override] + return self.last_level + + @classmethod + def from_json(cls, token_transfer_json: dict[str, Any]) -> 'TzktTokenBalanceData': + """Convert raw token transfer message from REST or WS into dataclass""" + token_json = token_transfer_json.get('token') or {} + standard = token_json.get('standard') + metadata = token_json.get('metadata') + contract_json = token_json.get('contract') or {} + + return TzktTokenBalanceData( + id=token_transfer_json['id'], + transfers_count=token_transfer_json['transfersCount'], + first_level=token_transfer_json['firstLevel'], + first_time=_parse_timestamp(token_transfer_json['firstTime']), + last_level=token_transfer_json['lastLevel'], + last_time=_parse_timestamp(token_transfer_json['lastTime']), + account_address=token_transfer_json.get('account', {}).get('address'), + account_alias=token_transfer_json.get('account', {}).get('alias'), + tzkt_token_id=token_json['id'], + contract_address=contract_json.get('address'), + contract_alias=contract_json.get('alias'), + token_id=token_json.get('tokenId'), + standard=TzktTokenStandard(standard) if standard else None, + metadata=metadata if isinstance(metadata, dict) else {}, + balance=token_transfer_json.get('balance'), + balance_value=token_transfer_json.get('balanceValue'), + ) + + @dataclass(frozen=True) class TzktEventData(HasLevel): """Basic structure for events received from TzKT REST API""" diff --git a/src/dipdup/project.py b/src/dipdup/project.py index 11c96abdb..58d46bad7 100644 --- a/src/dipdup/project.py +++ b/src/dipdup/project.py @@ -263,10 +263,17 @@ def render_project( def render_base( answers: Answers, force: bool = False, + include: set[str] | None = None, ) -> None: """Render base from template""" # NOTE: Common base - _render_templates(answers, Path('base'), force, refresh=True) + _render_templates( + answers=answers, + path=Path('base'), + force=force, + include=include, + exists=True, + ) _render( answers, @@ -276,7 +283,13 @@ def render_base( ) -def _render_templates(answers: Answers, path: Path, force: bool = False, refresh: bool = False) -> None: +def _render_templates( + answers: Answers, + path: Path, + force: bool = False, + include: set[str] | None = None, + exists: bool = False, +) -> None: from jinja2 import Template project_path = Path(__file__).parent / 'projects' / path @@ -284,7 +297,11 @@ def _render_templates(answers: Answers, path: Path, force: bool = False, refresh for path in project_paths: template_path = path.relative_to(Path(__file__).parent) - output_base = get_package_path(answers['package']) if refresh else Path(answers['package']) + + if include and not any(str(path).startswith(i) for i in include): + continue + + output_base = get_package_path(answers['package']) if exists else Path(answers['package']) output_path = Path( output_base, *path.relative_to(project_path).parts, diff --git a/src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 b/src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 new file mode 100644 index 000000000..757f20925 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/dipdup.yaml.j2 @@ -0,0 +1,21 @@ +spec_version: 2.0 +package: {{ project.package }} + +contracts: + tzbtc_mainnet: + kind: tezos + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: tzbtc + +datasources: + tzkt: + kind: tezos.tzkt + url: https://api.tzkt.io + +indexes: + tzbtc_holders_mainnet: + kind: tezos.tzkt.token_balances + datasource: tzkt + handlers: + - callback: on_balance_update + contract: tzbtc_mainnet diff --git a/src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 b/src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 new file mode 100644 index 000000000..b4a13aee6 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/handlers/on_balance_update.py.j2 @@ -0,0 +1,13 @@ +from decimal import Decimal +from {{ project.package }} import models as models +from dipdup.context import HandlerContext +from dipdup.models.tezos_tzkt import TzktTokenBalanceData + + +async def on_balance_update( + ctx: HandlerContext, + token_balance: TzktTokenBalanceData, +) -> None: + holder, _ = await models.Holder.get_or_create(address=token_balance.contract_address) + holder.balance = Decimal(token_balance.balance_value or 0) / (10**8) + await holder.save() \ No newline at end of file diff --git a/src/dipdup/projects/demo_token_balances/models/__init__.py.j2 b/src/dipdup/projects/demo_token_balances/models/__init__.py.j2 new file mode 100644 index 000000000..876bfdd21 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/models/__init__.py.j2 @@ -0,0 +1,8 @@ +from dipdup import fields + +from dipdup.models import Model + + +class Holder(Model): + address = fields.TextField(pk=True) + balance = fields.DecimalField(decimal_places=8, max_digits=20, default=0) diff --git a/src/dipdup/projects/demo_token_balances/replay.yaml b/src/dipdup/projects/demo_token_balances/replay.yaml new file mode 100644 index 000000000..8c8703a05 --- /dev/null +++ b/src/dipdup/projects/demo_token_balances/replay.yaml @@ -0,0 +1,5 @@ +spec_version: 2.0 +replay: + description: TzBTC FA1.2 token balances + package: demo_token_balances + template: demo_token_balances \ No newline at end of file diff --git a/src/dipdup/test.py b/src/dipdup/test.py index a71ec9957..ec5262d85 100644 --- a/src/dipdup/test.py +++ b/src/dipdup/test.py @@ -1,10 +1,30 @@ +"""This module contains helper functions for testing DipDup projects. + +These helpers are not part of the public API and can be changed without prior notice. +""" import asyncio +import atexit +import os +import tempfile +from collections.abc import AsyncIterator from contextlib import AsyncExitStack +from contextlib import asynccontextmanager +from pathlib import Path +from shutil import which +from typing import TYPE_CHECKING from typing import Any from dipdup.config import DipDupConfig +from dipdup.config import HasuraConfig +from dipdup.config import PostgresDatabaseConfig from dipdup.dipdup import DipDup +from dipdup.exceptions import FrameworkException from dipdup.index import Index +from dipdup.project import get_default_answers +from dipdup.yaml import DipDupYAMLConfig + +if TYPE_CHECKING: + from docker.client import DockerClient # type: ignore[import-untyped] async def create_dummy_dipdup( @@ -47,3 +67,142 @@ async def spawn_index(dipdup: DipDup, name: str) -> Index[Any, Any, Any]: index: Index[Any, Any, Any] = await dispatcher._ctx._spawn_index(name) dispatcher._indexes[name] = dispatcher._ctx._pending_indexes.pop() return index + + +def get_docker_client() -> 'DockerClient': + """Get Docker client instance if socket is available; skip test otherwise.""" + import _pytest.outcomes + from docker.client import DockerClient + + docker_socks = ( + Path('/var/run/docker.sock'), + Path.home() / 'Library' / 'Containers' / 'com.docker.docker' / 'Data' / 'vms' / '0' / 'docker.sock', + Path.home() / 'Library' / 'Containers' / 'com.docker.docker' / 'Data' / 'docker.sock', + ) + for path in docker_socks: + if path.exists(): + return DockerClient(base_url=f'unix://{path}') + + raise _pytest.outcomes.Skipped( # pragma: no cover + 'Docker socket not found', + allow_module_level=True, + ) + + +async def run_postgres_container() -> PostgresDatabaseConfig: + """Run Postgres container (destroyed on exit) and return database config with its IP.""" + docker = get_docker_client() + postgres_container = docker.containers.run( + image=get_default_answers()['postgres_image'], + environment={ + 'POSTGRES_USER': 'test', + 'POSTGRES_PASSWORD': 'test', + 'POSTGRES_DB': 'test', + }, + detach=True, + remove=True, + ) + atexit.register(postgres_container.stop) + postgres_container.reload() + postgres_ip = postgres_container.attrs['NetworkSettings']['IPAddress'] + + while not postgres_container.exec_run('pg_isready').exit_code == 0: + await asyncio.sleep(0.1) + + return PostgresDatabaseConfig( + kind='postgres', + host=postgres_ip, + port=5432, + user='test', + database='test', + password='test', + ) + + +async def run_hasura_container(postgres_ip: str) -> HasuraConfig: + """Run Hasura container (destroyed on exit) and return config with its IP.""" + docker = get_docker_client() + hasura_container = docker.containers.run( + image=get_default_answers()['hasura_image'], + environment={ + 'HASURA_GRAPHQL_DATABASE_URL': f'postgres://test:test@{postgres_ip}:5432', + }, + detach=True, + remove=True, + ) + atexit.register(hasura_container.stop) + hasura_container.reload() + hasura_ip = hasura_container.attrs['NetworkSettings']['IPAddress'] + + return HasuraConfig( + url=f'http://{hasura_ip}:8080', + source='new_source', + create_source=True, + ) + + +@asynccontextmanager +async def tmp_project( + config_paths: list[Path], + package: str, + exists: bool, + env: dict[str, str] | None = None, +) -> AsyncIterator[tuple[Path, dict[str, str]]]: + """Create a temporary isolated DipDup project.""" + with tempfile.TemporaryDirectory() as tmp_package_path: + # NOTE: Dump config + config, _ = DipDupYAMLConfig.load(config_paths, environment=False) + tmp_config_path = Path(tmp_package_path) / 'dipdup.yaml' + tmp_config_path.write_text(config.dump()) + + # NOTE: Symlink packages and executables + tmp_bin_path = Path(tmp_package_path) / 'bin' + tmp_bin_path.mkdir() + for executable in ('dipdup', 'datamodel-codegen'): + if (executable_path := which(executable)) is None: + raise FrameworkException(f'Executable `{executable}` not found') # pragma: no cover + os.symlink(executable_path, tmp_bin_path / executable) + + os.symlink( + Path(__file__).parent.parent / 'dipdup', + Path(tmp_package_path) / 'dipdup', + ) + + # NOTE: Ensure that `run` uses existing package and `init` creates a new one + if exists: + os.symlink( + Path(__file__).parent.parent / package, + Path(tmp_package_path) / package, + ) + + # NOTE: Prepare environment + env = { + **os.environ, + **(env or {}), + 'PATH': str(tmp_bin_path), + 'PYTHONPATH': str(tmp_package_path), + 'DIPDUP_TEST': '1', + 'DIPDUP_DEBUG': '1', + } + + yield Path(tmp_package_path), env + + +async def run_in_tmp( + tmp_path: Path, + env: dict[str, str], + *args: str, +) -> None: + """Run DipDup in existing temporary project.""" + tmp_config_path = Path(tmp_path) / 'dipdup.yaml' + + proc = await asyncio.subprocess.create_subprocess_shell( + f'dipdup -c {tmp_config_path} {" ".join(args)}', + cwd=tmp_path, + shell=True, + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.communicate() + assert proc.returncode == 0 diff --git a/tests/__init__.py b/tests/__init__.py index a3f76569c..48e92b0f7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -10,9 +10,7 @@ env.set_test() -CONFIGS_PATH = Path(__file__).parent / 'configs' -REPLAYS_PATH = Path(__file__).parent / 'replays' -SRC_PATH = Path(__file__).parent.parent / 'src' +TEST_CONFIGS = Path(__file__).parent / 'configs' @asynccontextmanager diff --git a/tests/configs/demo_token_balances.yml b/tests/configs/demo_token_balances.yml new file mode 100644 index 000000000..3eabede72 --- /dev/null +++ b/tests/configs/demo_token_balances.yml @@ -0,0 +1,23 @@ +spec_version: 2.0 +package: demo_token_balances + +contracts: + tzbtc_mainnet: + kind: tezos + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: tzbtc + +datasources: + tzkt: + kind: tezos.tzkt + url: https://api.tzkt.io + +indexes: + tzbtc_holders_mainnet: + kind: tezos.tzkt.token_balances + datasource: tzkt + first_level: 1366824 + last_level: 1366999 + handlers: + - callback: on_balance_update + contract: tzbtc_mainnet \ No newline at end of file diff --git a/tests/configs/test_postgres.yaml b/tests/configs/test_postgres.yaml new file mode 100644 index 000000000..98615dd3e --- /dev/null +++ b/tests/configs/test_postgres.yaml @@ -0,0 +1,6 @@ +database: + kind: postgres + host: ${POSTGRES_HOST} + user: test + password: test + database: test \ No newline at end of file diff --git a/tests/configs/test_postgres_immune.yaml b/tests/configs/test_postgres_immune.yaml new file mode 100644 index 000000000..8fd521d66 --- /dev/null +++ b/tests/configs/test_postgres_immune.yaml @@ -0,0 +1,11 @@ +database: + kind: postgres + host: ${POSTGRES_HOST} + user: test + password: test + database: test + + immune_tables: + - tld + - domain + - test diff --git a/tests/configs/sqlite.yaml b/tests/configs/test_sqlite.yaml similarity index 100% rename from tests/configs/sqlite.yaml rename to tests/configs/test_sqlite.yaml diff --git a/tests/configs/test_sqlite_immune.yaml b/tests/configs/test_sqlite_immune.yaml new file mode 100644 index 000000000..95232dd9e --- /dev/null +++ b/tests/configs/test_sqlite_immune.yaml @@ -0,0 +1,11 @@ +database: + kind: sqlite + path: db.sqlite3 + + immune_tables: + - tld + - domain + - test + +advanced: + unsafe_sqlite: true \ No newline at end of file diff --git a/tests/test_demos.py b/tests/test_demos.py index 93d810cb9..7e5043f73 100644 --- a/tests/test_demos.py +++ b/tests/test_demos.py @@ -1,80 +1,16 @@ -import os -import subprocess -import tempfile -from collections.abc import AsyncIterator from collections.abc import Awaitable from collections.abc import Callable -from contextlib import AbstractAsyncContextManager from contextlib import AsyncExitStack -from contextlib import asynccontextmanager from decimal import Decimal from functools import partial -from pathlib import Path -from shutil import which import pytest -from dipdup.database import get_connection from dipdup.database import tortoise_wrapper -from dipdup.exceptions import FrameworkException from dipdup.models.tezos_tzkt import TzktOperationType -from tests import CONFIGS_PATH -from tests import SRC_PATH - - -@asynccontextmanager -async def tmp_project(config_path: Path, package: str, exists: bool) -> AsyncIterator[tuple[Path, dict[str, str]]]: - with tempfile.TemporaryDirectory() as tmp_package_path: - # NOTE: Symlink configs, packages and executables - tmp_config_path = Path(tmp_package_path) / 'dipdup.yaml' - os.symlink(config_path, tmp_config_path) - - tmp_bin_path = Path(tmp_package_path) / 'bin' - tmp_bin_path.mkdir() - for executable in ('dipdup', 'datamodel-codegen'): - if (executable_path := which(executable)) is None: - raise FrameworkException(f'Executable `{executable}` not found') - os.symlink(executable_path, tmp_bin_path / executable) - - os.symlink( - SRC_PATH / 'dipdup', - Path(tmp_package_path) / 'dipdup', - ) - - # NOTE: Ensure that `run` uses existing package and `init` creates a new one - if exists: - os.symlink( - SRC_PATH / package, - Path(tmp_package_path) / package, - ) - - # NOTE: Prepare environment - env = { - **os.environ, - 'PATH': str(tmp_bin_path), - 'PYTHONPATH': str(tmp_package_path), - 'DIPDUP_TEST': '1', - } - - yield Path(tmp_package_path), env - - -async def run_in_tmp( - tmp_path: Path, - env: dict[str, str], - *cmd: str, -) -> None: - sqlite_config_path = Path(__file__).parent / 'configs' / 'sqlite.yaml' - tmp_config_path = Path(tmp_path) / 'dipdup.yaml' - - subprocess.run( - f'dipdup -c {tmp_config_path} -c {sqlite_config_path} {" ".join(cmd)}', - cwd=tmp_path, - check=True, - shell=True, - env=env, - capture_output=True, - ) +from dipdup.test import run_in_tmp +from dipdup.test import tmp_project +from tests import TEST_CONFIGS async def assert_run_token() -> None: @@ -129,6 +65,18 @@ async def assert_run_token_transfers(expected_holders: int, expected_balance: st assert f'{random_balance:f}' == expected_balance +async def assert_run_balances() -> None: + import demo_token_balances.models + + holders = await demo_token_balances.models.Holder.filter().count() + holder = await demo_token_balances.models.Holder.first() + assert holder + random_balance = holder.balance + + assert holders == 1 + assert random_balance == 0 + + async def assert_run_big_maps() -> None: import demo_big_maps.models @@ -235,6 +183,8 @@ async def assert_run_dao() -> None: 'run', partial(assert_run_token_transfers, 2, '-0.02302128'), ), + ('demo_token_balances.yml', 'demo_token_balances', 'run', assert_run_balances), + ('demo_token_balances.yml', 'demo_token_balances', 'init', partial(assert_init, 'demo_token_balances')), ('demo_big_maps.yml', 'demo_big_maps', 'run', assert_run_big_maps), ('demo_big_maps.yml', 'demo_big_maps', 'init', partial(assert_init, 'demo_big_maps')), ('demo_domains.yml', 'demo_domains', 'run', assert_run_domains), @@ -262,11 +212,13 @@ async def test_run_init( cmd: str, assert_fn: Callable[[], Awaitable[None]], ) -> None: - config_path = CONFIGS_PATH / config + config_path = TEST_CONFIGS / config + env_config_path = TEST_CONFIGS / 'test_sqlite.yaml' + async with AsyncExitStack() as stack: tmp_package_path, env = await stack.enter_async_context( tmp_project( - config_path, + [config_path, env_config_path], package, exists=cmd != 'init', ), @@ -280,47 +232,3 @@ async def test_run_init( ) await assert_fn() - - -async def _count_tables() -> int: - conn = get_connection() - _, res = await conn.execute_query('SELECT count(name) FROM sqlite_master WHERE type = "table";') - return int(res[0][0]) - - -async def test_schema() -> None: - package = 'demo_token' - config_path = CONFIGS_PATH / f'{package}.yml' - - async with AsyncExitStack() as stack: - tmp_package_path, env = await stack.enter_async_context( - tmp_project( - config_path, - package, - exists=True, - ), - ) - - def tortoise() -> AbstractAsyncContextManager[None]: - return tortoise_wrapper( - f'sqlite://{tmp_package_path}/db.sqlite3', - f'{package}.models', - ) - - async with tortoise(): - conn = get_connection() - assert (await _count_tables()) == 0 - - await run_in_tmp(tmp_package_path, env, 'schema', 'init') - - async with tortoise(): - conn = get_connection() - assert (await _count_tables()) == 10 - await conn.execute_script('CREATE TABLE test (id INTEGER PRIMARY KEY);') - assert (await _count_tables()) == 11 - - await run_in_tmp(tmp_package_path, env, 'schema', 'wipe', '--force') - - async with tortoise(): - conn = get_connection() - assert (await _count_tables()) == 0 diff --git a/tests/test_hasura.py b/tests/test_hasura.py index ea233c607..bd48ce3f7 100644 --- a/tests/test_hasura.py +++ b/tests/test_hasura.py @@ -1,5 +1,3 @@ -import asyncio -import atexit import os from contextlib import AsyncExitStack from pathlib import Path @@ -9,7 +7,6 @@ import pytest from aiohttp import web from aiohttp.pytest_plugin import AiohttpClient -from docker.client import DockerClient # type: ignore[import-untyped] from tortoise import Tortoise from dipdup.config import DipDupConfig @@ -20,76 +17,14 @@ from dipdup.hasura import HasuraGateway from dipdup.models import ReindexingAction from dipdup.models import ReindexingReason -from dipdup.project import get_default_answers from dipdup.test import create_dummy_dipdup +from dipdup.test import run_hasura_container +from dipdup.test import run_postgres_container if TYPE_CHECKING: from aiohttp.test_utils import TestClient -def get_docker_client() -> DockerClient: - docker_socks = ( - Path('/var/run/docker.sock'), - Path.home() / 'Library' / 'Containers' / 'com.docker.docker' / 'Data' / 'vms' / '0' / 'docker.sock', - Path.home() / 'Library' / 'Containers' / 'com.docker.docker' / 'Data' / 'docker.sock', - ) - for path in docker_socks: - if path.exists(): - return DockerClient(base_url=f'unix://{path}') - else: - pytest.skip('Docker socket not found', allow_module_level=True) - - -async def run_postgres_container() -> PostgresDatabaseConfig: - docker = get_docker_client() - postgres_container = docker.containers.run( - image=get_default_answers()['postgres_image'], - environment={ - 'POSTGRES_USER': 'test', - 'POSTGRES_PASSWORD': 'test', - 'POSTGRES_DB': 'test', - }, - detach=True, - remove=True, - ) - atexit.register(postgres_container.stop) - postgres_container.reload() - postgres_ip = postgres_container.attrs['NetworkSettings']['IPAddress'] - - while not postgres_container.exec_run('pg_isready').exit_code == 0: - await asyncio.sleep(0.1) - - return PostgresDatabaseConfig( - kind='postgres', - host=postgres_ip, - port=5432, - user='test', - database='test', - password='test', - ) - - -async def run_hasura_container(postgres_ip: str) -> HasuraConfig: - docker = get_docker_client() - hasura_container = docker.containers.run( - image=get_default_answers()['hasura_image'], - environment={ - 'HASURA_GRAPHQL_DATABASE_URL': f'postgres://test:test@{postgres_ip}:5432', - }, - detach=True, - remove=True, - ) - atexit.register(hasura_container.stop) - hasura_container.reload() - hasura_ip = hasura_container.attrs['NetworkSettings']['IPAddress'] - - return HasuraConfig( - url=f'http://{hasura_ip}:8080', - source='new_source', - create_source=True, - ) - - async def test_configure_hasura() -> None: if os.uname().sysname != 'Linux' or 'microsoft' in os.uname().release: # check for WSL, Windows, mac and else pytest.skip('Test is not supported for os archetecture', allow_module_level=True) diff --git a/tests/test_index/test_tzkt_operations.py b/tests/test_index/test_tzkt_operations.py index 3e4032e4a..dbb2a28a3 100644 --- a/tests/test_index/test_tzkt_operations.py +++ b/tests/test_index/test_tzkt_operations.py @@ -19,7 +19,7 @@ from dipdup.models.tezos_tzkt import TzktOperationType from dipdup.test import create_dummy_dipdup from dipdup.test import spawn_index -from tests import CONFIGS_PATH +from tests import TEST_CONFIGS from tests import tzkt_replay @@ -31,7 +31,7 @@ async def tzkt() -> AsyncIterator[TzktDatasource]: @pytest.fixture def index_config() -> TzktOperationsIndexConfig: - config = DipDupConfig.load([CONFIGS_PATH / 'operation_filters.yml'], True) + config = DipDupConfig.load([TEST_CONFIGS / 'operation_filters.yml'], True) config.initialize() return cast(TzktOperationsIndexConfig, config.indexes['test']) @@ -124,7 +124,7 @@ async def test_get_transaction_filters(tzkt: TzktDatasource, index_config: TzktO async def test_get_sync_level() -> None: - config = DipDupConfig.load([CONFIGS_PATH / 'demo_token.yml'], True) + config = DipDupConfig.load([TEST_CONFIGS / 'demo_token.yml'], True) async with AsyncExitStack() as stack: dipdup = await create_dummy_dipdup(config, stack) index = await spawn_index(dipdup, 'tzbtc_holders_mainnet') @@ -149,7 +149,7 @@ async def test_get_sync_level() -> None: async def test_realtime() -> None: from demo_token import models - config = DipDupConfig.load([CONFIGS_PATH / 'demo_token.yml'], True) + config = DipDupConfig.load([TEST_CONFIGS / 'demo_token.yml'], True) async with AsyncExitStack() as stack: dipdup = await create_dummy_dipdup(config, stack) await dipdup._set_up_datasources(stack) diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 000000000..da62ed023 --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,183 @@ +from contextlib import AbstractAsyncContextManager +from contextlib import AsyncExitStack + +from dipdup.database import get_connection +from dipdup.database import get_tables +from dipdup.database import tortoise_wrapper +from dipdup.test import run_in_tmp +from dipdup.test import run_postgres_container +from dipdup.test import tmp_project +from tests import TEST_CONFIGS + +_dipdup_tables = { + 'dipdup_contract_metadata', + 'dipdup_model_update', + 'dipdup_schema', + 'dipdup_contract', + 'dipdup_token_metadata', + 'dipdup_head', + 'dipdup_index', + 'dipdup_meta', +} + + +async def test_schema_sqlite() -> None: + package = 'demo_domains' + config_path = TEST_CONFIGS / f'{package}.yml' + env_config_path = TEST_CONFIGS / 'test_sqlite.yaml' + + async with AsyncExitStack() as stack: + tmp_package_path, env = await stack.enter_async_context( + tmp_project( + [config_path, env_config_path], + package, + exists=True, + ), + ) + + def tortoise() -> AbstractAsyncContextManager[None]: + return tortoise_wrapper( + f'sqlite://{tmp_package_path}/db.sqlite3', + f'{package}.models', + ) + + async with tortoise(): + conn = get_connection() + assert await get_tables() == set() + + await run_in_tmp(tmp_package_path, env, 'schema', 'init') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain', 'sqlite_sequence'} + await conn.execute_script('CREATE TABLE test (id INTEGER PRIMARY KEY);') + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain', 'sqlite_sequence', 'test'} + + await run_in_tmp(tmp_package_path, env, 'schema', 'wipe', '--force') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == set() + + +async def test_schema_sqlite_immune() -> None: + package = 'demo_domains' + config_path = TEST_CONFIGS / f'{package}.yml' + env_config_path = TEST_CONFIGS / 'test_sqlite_immune.yaml' + + async with AsyncExitStack() as stack: + tmp_package_path, env = await stack.enter_async_context( + tmp_project( + [config_path, env_config_path], + package, + exists=True, + ), + ) + + def tortoise() -> AbstractAsyncContextManager[None]: + return tortoise_wrapper( + f'sqlite://{tmp_package_path}/db.sqlite3', + f'{package}.models', + ) + + async with tortoise(): + conn = get_connection() + assert await get_tables() == set() + + await run_in_tmp(tmp_package_path, env, 'schema', 'init') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain', 'sqlite_sequence'} + await conn.execute_script('CREATE TABLE test (id INTEGER PRIMARY KEY);') + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain', 'sqlite_sequence', 'test'} + + await run_in_tmp(tmp_package_path, env, 'schema', 'wipe', '--force') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == {'dipdup_meta', 'test', 'domain', 'tld'} + + +async def test_schema_postgres() -> None: + package = 'demo_domains' + config_path = TEST_CONFIGS / f'{package}.yml' + env_config_path = TEST_CONFIGS / 'test_postgres.yaml' + + async with AsyncExitStack() as stack: + tmp_package_path, env = await stack.enter_async_context( + tmp_project( + [config_path, env_config_path], + package, + exists=True, + ), + ) + + database_config = await run_postgres_container() + env['POSTGRES_HOST'] = database_config.host + + def tortoise() -> AbstractAsyncContextManager[None]: + return tortoise_wrapper( + database_config.connection_string, + f'{package}.models', + ) + + async with tortoise(): + conn = get_connection() + assert await get_tables() == set() + + await run_in_tmp(tmp_package_path, env, 'schema', 'init') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain'} + await conn.execute_script('CREATE TABLE test (id INTEGER PRIMARY KEY);') + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain', 'test'} + + await run_in_tmp(tmp_package_path, env, 'schema', 'wipe', '--force') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == {'dipdup_meta'} + + +async def test_schema_postgres_immune() -> None: + package = 'demo_domains' + config_path = TEST_CONFIGS / f'{package}.yml' + env_config_path = TEST_CONFIGS / 'test_postgres_immune.yaml' + + async with AsyncExitStack() as stack: + tmp_package_path, env = await stack.enter_async_context( + tmp_project( + [config_path, env_config_path], + package, + exists=True, + ), + ) + + database_config = await run_postgres_container() + env['POSTGRES_HOST'] = database_config.host + + def tortoise() -> AbstractAsyncContextManager[None]: + return tortoise_wrapper( + database_config.connection_string, + f'{package}.models', + ) + + async with tortoise(): + conn = get_connection() + assert await get_tables() == set() + + await run_in_tmp(tmp_package_path, env, 'schema', 'init') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain'} + await conn.execute_script('CREATE TABLE test (id INTEGER PRIMARY KEY);') + assert await get_tables() == _dipdup_tables | {'tld', 'record', 'domain', 'test'} + + await run_in_tmp(tmp_package_path, env, 'schema', 'wipe', '--force') + + async with tortoise(): + conn = get_connection() + assert await get_tables() == {'dipdup_meta', 'test', 'domain', 'tld'}