wip full deploy #22
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Forest-Loss-Driver-Prediction | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - henryh/favyen/forest-loss-20240917 | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| SERVICE_NAME: "rslearn_projects" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| outputs: | |
| ghcr_docker_image: ${{ steps.image-names.outputs.ghcr_image_name }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| tags: | | |
| type=sha,format=long | |
| type=sha,format=short | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| - name: Cleanup disk space | |
| run: | | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| - name: Build and push Docker image | |
| id: build-push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| # cache-from: type=gha | |
| # cache-to: type=gha,mode=max | |
| build-args: | | |
| GIT_USERNAME=${{ secrets.GIT_USERNAME }} | |
| GIT_TOKEN=${{ secrets.GIT_TOKEN }} | |
| - name: Store Image Names | |
| id: image-names | |
| run: |- | |
| GHCR_IMAGE="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-push.outputs.digest }}" | |
| GHCR_IMAGE=`echo ${GHCR_IMAGE} | tr '[:upper:]' '[:lower:]'` # docker requires that all image names be lowercase | |
| echo "ghcr.io Docker image name is ${GHCR_IMAGE}" | |
| echo "ghcr_image_name=\"${GHCR_IMAGE}\"" >> $GITHUB_OUTPUT | |
| predict: | |
| runs-on: ubuntu-latest-m | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Cleanup disk space | |
| run: | | |
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | |
| sudo docker image prune --all --force >/dev/null 2>&1 || true | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| - name: Authenticate into gcp | |
| uses: "google-github-actions/auth@v2" | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} | |
| - name: Debug script location | |
| run: | | |
| pwd | |
| ls -la | |
| - name: Run Extract Dataset Job on a VM and Launch Prediction Job on Beaker # We need all of these to become secrets | |
| run: | | |
| export GHCR_PAT=${{ secrets.GHCR_PAT_PULL_DOCKER_IMAGE }} && \ | |
| export DATASET_EXTRACT_COMMAND="python -m rslp.main forest_loss_driver extract_dataset" && \ | |
| export RSLP_PROJECT="forest_loss_driver" && \ | |
| bash .github/workflows/deploy_image_on_vm.sh \ | |
| --project-id ${{ secrets.GCP_PROJECT_ID }} \ | |
| --zone "us-west1-b" \ | |
| --machine-type "n2-standard-8" \ | |
| --docker-image ${{ steps.image-names.outputs.ghcr_image_name }} \ | |
| --command "$DATASET_EXTRACT_COMMAND" \ | |
| --user ${{ secrets.GCP_USER }} \ | |
| --ghcr-user ${{ secrets.GHCR_USER }} \ | |
| --service-account ${{ secrets.FOREST_LOSS_DRIVER_INFERENCE_SERVICE_ACCOUNT }} \ | |
| --delete no \ | |
| --beaker-token ${{ secrets.BEAKER_TOKEN }} \ | |
| --beaker-addr ${{ secrets.BEAKER_ADDR }} \ | |
| --beaker-username ${{ secrets.BEAKER_USERNAME }} \ | |
| --rslp-project $RSLP_PROJECT \ | |
| --workflow predict \ | |
| --rslp-prefix ${{ secrets.RSLP_PREFIX }} \ | |
| --gpu-count 1 \ | |
| --shared-memory "64Gib" \ | |
| --cluster ${{ secrets.BEAKER_CLUSTER_INFERENCE }} \ | |
| --priority "normal" \ | |
| --task-name "${RSLP_PROJECT}_inference_$(uuidgen | cut -c1-8)" \ | |
| --budget ${{ secrets.BEAKER_BUDGET }} \ | |
| --workspace ${{ secrets.BEAKER_WORKSPACE }} | |
| # FOR clean up i will just run this vm with an instance schedule for like a day or something and have it log how long everything took somewhere | |
| # I want to just launch predict as a single beaker job and have it run on a gpu |