build_test.yaml

name: Build, Test, and Publish Docker Image

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
  SERVICE_NAME: "rslearn_projects"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    outputs:
        ghcr_docker_image: ${{ steps.image-names.outputs.ghcr_image_name }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=sha,format=long
            type=sha,format=short
            type=raw,value=latest,enable={{is_default_branch}}

      - name: Cleanup disk space
        run: |
          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
          sudo docker image prune --all --force >/dev/null 2>&1 || true
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost

      - name: Build and push Docker image
        id: build-push
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            GIT_USERNAME=${{ secrets.GIT_USERNAME }}
            GIT_TOKEN=${{ secrets.GIT_TOKEN }}

      - name: Store Image Names
        # We need the docker image name downstream in test & deploy. This saves the full docker image names to outputs
        id: image-names
        run: |-
          GHCR_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}"
          GHCR_IMAGE=`echo ${GHCR_IMAGE} | tr '[:upper:]' '[:lower:]'` # docker requires that all image names be lowercase
          echo "ghcr.io Docker image name is ${GHCR_IMAGE}"
          echo "ghcr_image_name=\"${GHCR_IMAGE}\"" >> $GITHUB_OUTPUT


  test:
    runs-on: ubuntu-latest-m
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
            registry: ${{ env.REGISTRY }}
            username: ${{ github.actor }}
            password: ${{ secrets.GITHUB_TOKEN }}

      - name: Cleanup disk space
        run: |
          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
          sudo docker image prune --all --force >/dev/null 2>&1 || true
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost

      - name: Build docker images
        run: |
          COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yaml build

      - name: Authenticate into gcp
        uses: "google-github-actions/auth@v2"
        with:
          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}

      - name: Run unit tests with Docker Compose
        run: |
          docker compose -f docker-compose.yaml run \
            test pytest tests/unit/

      - name: Run tests with Docker Compose
        run: |
          docker compose -f docker-compose.yaml run \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -v ${{env.GOOGLE_GHA_CREDS_PATH}}:/tmp/gcp-credentials.json:ro \
            -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json \
            -e RSLP_BUCKET=rslearn-eai \
            -e RSLP_PREFIX=gs://rslearn-eai \
            test pytest tests/integration/ --ignore tests/integration_slow/ -vv


      - name: Clean up
        if: always()
        run: |
          docker compose -f docker-compose.yaml down

  test-slow:
    runs-on: GPU-Enabled-Runner
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Log in to the Container registry
        uses: docker/login-action@v3
        with:
            registry: ${{ env.REGISTRY }}
            username: ${{ github.actor }}
            password: ${{ secrets.GITHUB_TOKEN }}

      - name: Cleanup disk space
        run: |
          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
          sudo docker image prune --all --force >/dev/null 2>&1 || true
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf /usr/local/share/boost

      - name: Build docker images
        run: |
          COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -f docker-compose.yaml build

      - name: Authenticate into gcp
        uses: "google-github-actions/auth@v2"
        with:
          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}

      - name: Run tests with Docker Compose
        run: |
          docker run \
            --gpus all \
            --shm-size=15g \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            -v ${{env.GOOGLE_GHA_CREDS_PATH}}:/tmp/gcp-credentials.json:ro \
            -e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp-credentials.json \
            -e RSLP_BUCKET=rslearn-eai \
            -e RSLP_PREFIX=gs://rslearn-eai \
            rslearn_projects-test pytest tests/integration_slow/ -vv

      - name: Clean up
        if: always()
        run: |
          docker compose -f docker-compose.yaml down