pex deploy using cli

.github/workflows/deploy.yml

@@ -4,25 +4,31 @@ on:
     branches:
       - "main"
       - "master"
+      - "benpankow/pex-deploy-using-cli"
 
 concurrency:
   # Cancel in-progress deploys to same branch
   group: ${{ github.ref }}/deploy
   cancel-in-progress: true
 env:
-  DAGSTER_CLOUD_URL: "http://ORGANIZATION_NAME.dagster.cloud"
+  DAGSTER_CLOUD_URL: "http://elementl-serverless-test.dagster.cloud"
   DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
-  ENABLE_FAST_DEPLOYS: 'true'
+  ENABLE_FAST_DEPLOYS: 'false'
   PYTHON_VERSION: '3.8'
+  DAGSTER_CLOUD_ORGANIZATION: "elementl-serverless-test"
   DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
+  # Path to the root folder containing the dagster project
+  DAGSTER_PROJECT_DIR: "."
+  # Path to dagster_cloud.yaml relative to DAGSTER_PROJECT_DIR
+  DAGSTER_CLOUD_YAML_PATH: "dagster_cloud.yaml"
 
 jobs:
   dagster_cloud_default_deploy:
     name: Dagster Serverless Deploy
     runs-on: ubuntu-20.04
     outputs:
       build_info: ${{ steps.parse-workspace.outputs.build_info }}
-    
+
     steps:
       - name: Prerun Checks
         id: prerun
@@ -35,46 +41,68 @@ jobs:
         with:
           dagster_cloud_file: $DAGSTER_CLOUD_FILE
 
-      - name: Checkout for Python Executable Deploy
-        if: steps.prerun.outputs.result == 'pex-deploy'
+      - name: Checkout
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
-          path: project-repo
-
-      - name: Python Executable Deploy
+
+      # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
+      - name: Initialize build session
+        id: ci-init
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
+          dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_YAML_PATH }}
+          # The full deployment name. If this run is for a PR, this value is ignored and a branch
+          # deployment is used.
+          deployment: 'prod'
+
+
+      # If using fast build, build the PEX
+      - name: Run PEX build
+        id: run-pex-build
         if: steps.prerun.outputs.result == 'pex-deploy'
-        uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
+        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1
         with:
-          dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
-          build_output_dir: "$GITHUB_WORKSPACE/build"
-          python_version: "${{ env.PYTHON_VERSION }}"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  dagster_cloud_docker_deploy:
-    name: Docker Deploy
-    runs-on: ubuntu-20.04
-    if: needs.dagster_cloud_default_deploy.outputs.build_info
-    needs: dagster_cloud_default_deploy
-    strategy:
-      fail-fast: false
-      matrix:
-        location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+          command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }}"
+
+      # Otherwise, set up and perform Docker build
+
+      # Any value can be used as the docker image tag. It is recommended to use a unique value
+      # for each build so that multiple builds do not overwrite each other.
+      - name: Generate docker image tag
+        id: generate-image-tag
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        run: echo "IMAGE_TAG=$GITHUB_SHA-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" >> $GITHUB_ENV && echo $IMAGE_TAG
+
+      # Enable buildx for caching
+      - name: Set up Docker Buildx
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and upload Docker image for "quickstart_etl"
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        uses: docker/build-push-action@v4
         with:
-          ref: ${{ github.head_ref }}
-      - name: Build and deploy to Dagster Cloud serverless
-        uses: dagster-io/dagster-cloud-action/actions/[email protected]
+          context: .
+          push: true
+          tags: ${{ env.IMAGE_REGISTRY }}:${{ env.IMAGE_TAG }}-quickstart_etl
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Update build session with image tag for quickstart_etl
+        id: ci-set-build-output-example-location
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
         with:
-          dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
-          location: ${{ toJson(matrix.location) }}
-          base_image: "python:${{ env.PYTHON_VERSION }}-slim"
-          # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs
-          # env_vars: ${{ toJson(secrets) }}
-          organization_id: ${{ secrets.ORGANIZATION_ID }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          command: "ci set-build-output --location-name=quickstart_etl --image-tag=$IMAGE_TAG-quickstart_etl"
 
+
+
+      # Deploy all code locations in this build session to Dagster Cloud
+      - name: Deploy to Dagster Cloud
+        id: ci-deploy
+        if: steps.prerun.outputs.result != 'skip'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          command: "ci deploy"