Merge branch 'next' into docs/glossary

.dockerignore

@@ -1,16 +1,15 @@
 # Ignore all
 *
 
-# Add build files
-!Makefile
+# Add metadata and build files
 !pyproject.toml
-!poetry.lock
+!pdm.lock
 !README.md
 !requirements**
 
-# Add code
-!src/dipdup/
+# Add Python packages
+!src/
 
 # Ignore caches
-**/.mypy_cache
+**/.*_cache
 **/__pycache__

.github/ISSUE_TEMPLATE/bug_report.md

@@ -17,7 +17,7 @@ about: Something doesn't work as expected
 * **Database engine version:**
 * **Hasura version:**
 * **Are you in docker?**
-* **Can you reproduce this issue on `master`?**
+* **Can you reproduce this issue on `next`?**
 
 <!---
 ### Attachment checklist

.github/workflows/build.yml

@@ -1,5 +1,5 @@
 name: Build
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 on:
@@ -12,7 +12,8 @@ on:
       - 'src/**'
       - 'Dockerfile'
       - 'pyproject.toml'
-      - 'poetry.lock'
+      # NOTE: Dumped with `pdm run update`
+      - 'requirements.txt'
       - '.github/workflows/build.yml'
 
 jobs:
@@ -25,26 +26,26 @@ jobs:
 
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
 
       - name: Log in to the registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.DOCKER_REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up metadata
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
           flavor: |
@@ -54,7 +55,7 @@ jobs:
             type=ref,event=tag
 
       - name: Publish nightly image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: Dockerfile

.github/workflows/docs.yml

@@ -1,65 +1,75 @@
+# NOTE: To run this pipeline in act replace secrets.GITHUB_TOKEN with $GITHUB_TOKEN and uncomment the first step
+
 name: Docs
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 on:
   push:
     tags:
-      - '*.*.*'
+      - '8.*.*'
+      - 'docs-next'
     branches:
       - '**'
     paths:
       - 'docs/**'
       - 'scripts/**'
       - '.github/workflows/docs.yml'
+      - '.github/workflows/installer.yml'
+
+env:
+  FRONTEND_BRANCH: dev
+  GITHUB_TOKEN: ${{ secrets.DOCS_GITHUB_TOKEN }}
 
 jobs:
   docs:
     name: Docs
     runs-on: ubuntu-22.04
     steps:
-    - name: Check out the repo
-      uses: actions/checkout@v3
+      # NOTE: Uncomment for local testing with act
+      # - name: Install GitHub CLI (act only)
+      #   run: |
+      #     type -p curl >/dev/null || (sudo apt update && sudo apt install curl -y)
+      #     curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \
+      #     && sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \
+      #     && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
+      #     && sudo apt update \
+      #     && sudo apt install gh -y
 
-    - name: Install poetry
-      run: pipx install poetry
+      - name: Check out the repo
+        uses: actions/checkout@v4
 
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.11'
-        cache: 'poetry'
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+          cache: 'pip'
 
-    - name: Set up mdbook
-      uses: jontze/action-mdbook@v2
-      with:
-        token: ${{secrets.GITHUB_TOKEN}}
-        use-toc: true
-        use-admonish: true
+      - name: Install PDM
+        run: pip install pdm
 
-    - name: Install project
-      run: make install
+      - name: Install markdownlint-cli
+        run: npm install -g markdownlint-cli
 
-    - name: Build docs (docs)
-      if: startsWith(github.ref, 'refs/tags') && !contains(github.ref, 'rc')
-      run: MDBOOK_ROOT=https://docs.dipdup.io make docs
+      - name: Install project
+        run: pdm sync
 
-    - name: Deploy to GH Pages (docs)
-      if: startsWith(github.ref, 'refs/tags') && !contains(github.ref, 'rc')
-      uses: peaceiris/actions-gh-pages@v3
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: ./docs/dist/docs
-        destination_dir: docs
+      - name: Clone frontend
+        run: |
+          gh auth setup-git
+          gh repo clone dipdup-io/interface ../interface -- --single-branch --branch "$FRONTEND_BRANCH"
 
-    - name: Build docs (docs-next)
-      if: startsWith(github.ref, 'refs/heads')
-      run: MDBOOK_ROOT=https://docs-next.dipdup.io make docs
+      - name: Build docs
+        run: pdm run make docs
 
-    - name: Deploy to GH Pages (docs-next)
-      if: startsWith(github.ref, 'refs/heads/master')
-      uses: peaceiris/actions-gh-pages@v3
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        publish_dir: ./docs/dist/docs
-        destination_dir: docs-next
+      # NOTE: Installer is tested in `installer` workflow
+      - name: Commit and push frontend
+        if: github.ref_type == 'tag'
+        run: |
+          cp src/dipdup/install.py ../interface/public/install.py
+          cd ../interface
+          git config --local user.email "[email protected]"
+          git config --local user.name "GitHub Action"
+          git add -A
+          git diff-index --quiet HEAD || git commit -m "${{ github.event.head_commit.message }}"
+          git push origin $FRONTEND_BRANCH

.github/workflows/installer.yml

@@ -1,14 +1,14 @@
 name: Installer
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 on:
   push:
     paths:
       - 'src/**'
-      - 'scripts/install.py'
       - 'pyproject.toml'
-      - 'poetry.lock'
+      - 'pdm.lock'
+      - '.github/workflows/docs.yml'
       - '.github/workflows/installer.yml'
 
 jobs:
@@ -28,12 +28,13 @@ jobs:
             arch: arm64
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.12'
+          cache: 'pip'
 
       - name: macOS runner workaround
         run: pip install --upgrade pip pipx
@@ -43,23 +44,14 @@ jobs:
 
       - name: dipdup new
         run: dipdup new --quiet
-
+        
       - name: dipdup init
-        run: cd dipdup-indexer; dipdup init
-
-      - name: make install
-        run: cd dipdup-indexer; make install
+        run: cd dipdup_indexer; dipdup init
 
-      - name: reinstall dipdup from HEAD
-        run: cd dipdup-indexer; poetry run pip install --upgrade --force-reinstall --no-deps --no-cache-dir ..
+      - name: Install dev dependencies
+        run: cd dipdup_indexer; pdm venv create; pdm remove dipdup; pdm add ..; pdm install
 
-      - name: make lint
-        run: cd dipdup-indexer; make lint
+      - name: make all
+        run: cd dipdup_indexer; $(pdm venv activate); make all
 
-      - name: Deploy scripts to GH Pages
-        if: contains(github.ref, 'master')
-        uses: peaceiris/actions-gh-pages@v3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./scripts
-          keep_files: true
+      # NOTE: Installer is published in `docs` workflow

.github/workflows/release.yml

@@ -8,32 +8,45 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-22.04
+    env:
+      DOCKER_REGISTRY: ghcr.io
+      DOCKER_IMAGE_NAME: ${{ github.repository }}
+
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         # NOTE: Fetch full history for Sentry release
         with:
           fetch-depth: 0
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
 
-      - name: Log in to the registry
-        uses: docker/login-action@v1
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Set up metadata
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker metadata
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
-          images: dipdup/dipdup
+          images: |
+            dipdup/dipdup
+            ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
           flavor: |
             latest=false
           tags: |
@@ -44,22 +57,22 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.12'
+          cache: 'pip'
 
-      - name: Set up Poetry
-        uses: snok/install-poetry@v1
-        with:
-          version: '1.4.0'
+      - name: Install PDM
+        run: pipx install pdm
+
+      - name: Run install
+        run: pdm install
 
-      - name: Install project
-        run: make install
-      - name: Run lint
-        run: make lint
-      - name: Run tests
-        run: make test
+      - name: Run all checks
+        run: pdm run make all
+        env:
+          NODE_API_KEY: ${{ secrets.ALCHEMY_KEY }}
 
       - name: Publish stable image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: Dockerfile
@@ -69,11 +82,9 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
 
       - name: Publish package on PyPi
-        run: |
-          poetry config http-basic.pypi __token__ ${{secrets.PYPI_TOKEN}}
-          poetry build
-          poetry publish
+        run: pdm publish --password ${{secrets.PYPI_TOKEN}}
 
+      # FIXME: Fails on prereleases; https://github.com/mindsers/changelog-reader-action/pull/39
       - name: Parse changelog
         id: changelog
         uses: mindsers/changelog-reader-action@v2
@@ -92,13 +103,14 @@ jobs:
             ## ${{ steps.changelog.outputs.version }} - ${{ steps.changelog.outputs.date }}
 
             ${{ steps.changelog.outputs.changes }}
-          draft: true
+          draft: false
           prerelease: ${{ steps.changelog.outputs.status == 'prereleased' }}
 
       - name: Create Sentry release
         uses: getsentry/action-release@v1
         with:
           version: ${{ steps.changelog.outputs.version }}
+          ignore_missing: true
         env:
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           SENTRY_ORG: sentry