test-build-pipeline.yml

name: Test and Build Pipeline

on:
    push:
      branches:
        - main
        - develop
      paths-ignore:
        - .github/**
        - docs/**
        - README.md
        - db.sqlite3.dist
    workflow_dispatch:

permissions:
  contents: write
  pull-requests: write

jobs:
    unit-tests:
      runs-on: ubuntu-latest
      steps:
        - name: Checkout the Code
          uses: actions/checkout@v4
        - name: Set Up Python
          uses: actions/setup-python@v5
          with:
            python-version: '3.12.8'
            cache: 'pip'
        - name: Install Swirl
          run: ./install.sh
        - name: Install the Unit Tests
          run: ./install-test.sh
        - name: Run the Unit Tests
          run: pytest

    db-dist:
      needs: unit-tests
      runs-on: ubuntu-latest
      concurrency: db-dist-cgos
      services:
        redis:
          image: redis
          options: >-
            --health-cmd "redis-cli ping"
            --health-interval 10s
            --health-timeout 5s
            --health-retries 5
          ports:
            - 6379:6379
      steps:
        - name: Checkout the Code
          uses: actions/checkout@v4
        - name: Pull the Latest (for just this branch)
          run: git pull origin ${{ github.ref_name }}
        - name: Set Up Python
          uses: actions/setup-python@v5
          with:
            python-version: '3.12.8'
            cache: 'pip'
        - name: Install Swirl (with a timeout)
          run: ./install.sh
          timeout-minutes: 10
        - name: Delete the db.sqlite3 File
          run: rm db.sqlite3
        - name: Setup Swirl
          run: python swirl.py setup
        - name: Create the Superuser
          run: DJANGO_SUPERUSER_PASSWORD=password python manage.py createsuperuser --email admin@example.com --username admin --noinput
        - name: Start Swirl
          run: python swirl.py start
        - name: Load the Default SearchProviders
          run:  python swirl_load.py -u admin -p password -s http://localhost:8000/ 'SearchProviders/preloaded.json'
        - name: Stop Swirl
          run: python swirl.py stop
        - name: Create a New db.dist File
          run: cp db.sqlite3 db.sqlite3.dist
        - name: Create a Pull Request
          id: createpr
          uses: peter-evans/create-pull-request@v7
          with:
            add-paths: db.sqlite3.dist
            branch: db_dist/autoupdate
            delete-branch: true
            commit-message: Update the db.sqlite3.dist file
            committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
            author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com>
            title: 'New db.sqlite3.dist generated by a db-dist.yml workflow'
            body: |
              New `db.sqlite3.dist` file generated by GitHub Action run ${{ github.run_number }}
        - name: Auto-Approve the PR
          if: steps.createpr.outputs.pull-request-operation == 'created'
          run: |
            echo "${{ secrets.GH_ACTIONS_TOKEN }}" | gh auth login --with-token
            gh pr review --approve "${{ steps.createpr.outputs.pull-request-number }}"
        - name: Auto-Merge the PR
          if: steps.createpr.outputs.pull-request-operation == 'created'
          run: |
            echo "${{ secrets.GH_ACTIONS_TOKEN }}" | gh auth login --with-token
            gh pr merge --merge --auto "${{ steps.createpr.outputs.pull-request-number }}"

    qa-suite:
      needs: db-dist
      runs-on: ubuntu-latest
      services:
        redis:
          image: redis
          options: >-
            --health-cmd "redis-cli ping"
            --health-interval 10s
            --health-timeout 5s
            --health-retries 5
          ports:
            - 6379:6379
      steps:
        - name: Checkout the Code
          uses: actions/checkout@v4
        - name: Pull the Latest (for just this branch)
          run: git pull origin ${{ github.ref_name }}
        - name: Login to Docker Hub
          uses: docker/login-action@v3
          with:
            username: ${{ secrets.SBS_DOCKER_USER }}
            password: ${{ secrets.SBS_DOCKER_PAT }}
        - name: Set Up Python
          uses: actions/setup-python@v5
          with:
            python-version: '3.12.8'
            cache: 'pip'
        - name: Install Swirl
          run: ./install.sh
        - name: Setup Swirl
          run: python swirl.py setup
        - name: Install the Galaxy UI Preview
          run: ./install-ui.sh -p
          env:
            MSAL_CB_PORT: 8000
            MSAL_HOST: localhost
        - name: Start Swirl
          run: |
            echo "OPENAI_API_KEY='${{ secrets.QA_OPENAI_KEY }}'" >> .env
            cat .env
            python swirl.py start
          env:
            ALLOWED_HOSTS: localhost,host.docker.internal
        - name: Decode BigQuery token and create JSON file
          run: |
            echo "$BIGQUERY_TOKEN_BASE64" | base64 --decode > "${{ github.workspace }}/token.json"
          env:
            BIGQUERY_TOKEN_BASE64: ${{ secrets.QA_BIGQUERY_TOKEN_BASE64 }}
        - name: Run the QA Suite
          run: |
            echo "SWIRL_TEST_HOST=localhost" > .env.qa
            echo "MSAL_HOST=localhost" >> .env.qa
            echo "MSAL_CB_PORT=8000" >> .env.qa
            echo "QA_ADMIN_PW=${{ secrets.QA_ADMIN_PW }}" >> .env.qa
            echo "QA_OPENAI_KEY=${{ secrets.QA_OPENAI_KEY }}" >> .env.qa
            echo "QA_NLR_USERNAME=${{ secrets.QA_NLR_USERNAME }}" >> .env.qa
            echo "QA_NLR_PASSWORD=${{ secrets.QA_NLR_PASSWORD }}" >> .env.qa
            echo "QA_CRUNCHBASE_KEY=${{ secrets.QA_CRUNCHBASE_KEY }}" >> .env.qa
            echo "QA_BLOCKCHAIN_KEY=${{ secrets.QA_BLOCKCHAIN_KEY }}" >> .env.qa
            echo "QA_YOUTRACK_TOKEN=${{ secrets.QA_YOUTRACK_TOKEN }}" >> .env.qa
            echo "QA_GITHUB_TOKEN=${{ secrets.QA_GITHUB_TOKEN }}" >> .env.qa
            echo "BIGQUERY_TOKEN_PATH=${{ github.workspace }}/token.json" >> .env.qa
            echo "QA_TRELLO_KEYS=${{ secrets.QA_TRELLO_KEYS }}" >> .env.qa
            echo "========"
            cat .env.qa
            echo "========"
            docker run --net=host --env-file .env.qa -t swirlai/swirl-search-qa:automated-tests-develop sh -c "behave --tags=qa_suite,community"

    swirl-docker:
      needs: qa-suite
      runs-on: ubuntu-latest
      steps:
        - name: Checkout the Code
          uses: actions/checkout@v4
        - name: Pull the Latest (for just this branch)
          run: git pull origin ${{ github.ref_name }}
        - name: Login to Docker Hub
          uses: docker/login-action@v3
          with:
            username: ${{ secrets.SBS_DOCKER_USER }}
            password: ${{ secrets.SBS_DOCKER_PAT }}
        - name: Build and Push Swirl Docker Image
          run: |
            BRANCH_NAME=${GITHUB_REF#refs/heads/}
            TAG_NAME=$([ "$BRANCH_NAME" = "main" ] && echo "latest" || echo "$BRANCH_NAME")
            docker buildx use devBuilder || docker buildx create --name devBuilder --use
            docker buildx build -t swirlai/swirl-search:$TAG_NAME --platform linux/amd64,linux/arm64 --push .
        - name: Update the Docker Repo Description
          uses: peter-evans/dockerhub-description@v4
          with:
            username: ${{ secrets.SBS_DOCKER_USER }}
            password: ${{ secrets.SBS_DOCKER_PAT }}
            repository: swirlai/swirl-search
        - name: Upload Log Files
          if: always()
          uses: actions/upload-artifact@v4
          with:
            name: log-files
            path: |
              logs/
              /var/log/syslog*