ci(workflow): add integration-test-latest|release and helm-integratio…

…n-test-latest|release workflow for mac
instill-ai · Sep 19, 2023 · 7b4581b · 7b4581b
1 parent 6e5a915
commit 7b4581b
Show file tree

Hide file tree

Showing 3 changed files with 249 additions and 4 deletions.
diff --git a/.github/workflows/helm-integration-test-backend.yml b/.github/workflows/helm-integration-test-backend.yml
@@ -11,7 +11,7 @@ on:
         type: string
 
 jobs:
-  helm-integration-test-latest:
+  helm-integration-test-latest-linux:
     if: inputs.target == 'latest'
     runs-on: ubuntu-latest
     steps:
@@ -108,7 +108,125 @@ jobs:
           cd ${{ inputs.component }}
           make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}
 
-  helm-integration-test-release:
+  helm-integration-test-latest-mac:
+    if: inputs.target == 'latest'
+    runs-on: [self-hosted,macOS,vdp]
+    steps:
+      - name: Set up environment
+        run: |
+          brew install make
+
+      - name: Checkout repo (vdp)
+        uses: actions/checkout@v3
+        with:
+          repository: instill-ai/vdp
+
+      - name: Load .env file (vdp)
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+
+      - name: Check if VDP Helm release exists
+        id: check-vdp-helm-release
+        run: |
+          if helm ls -n instill-ai | grep -q 'vdp'; then
+            echo "Helm release 'vdp' found."
+            echo "release_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "Helm release 'vdp' not found."
+          fi
+
+      - name: Uninstall VDP Helm Release
+        if: steps.check-vdp-helm-release.outputs.release_exists == 'true'
+        run: |
+          helm uninstall vdp --namespace instill-ai
+
+      - name: Install k6
+        run: |
+          brew install k6
+
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          repository: instill-ai/base
+
+      - name: Load .env file
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+
+      - name: Check if Base Helm release exists
+        id: check-base-helm-release
+        run: |
+          if helm ls -n instill-ai | grep -q 'base'; then
+            echo "Helm release 'base' found."
+            echo "release_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "Helm release 'base' not found."
+          fi
+
+      - name: Uninstall Base Helm Release
+        if: steps.check-base-helm-release.outputs.release_exists == 'true'
+        run: |
+          helm uninstall base --namespace instill-ai
+          kubectl delete namespace instill-ai
+
+      - name: Launch Helm Instill Base (latest)
+        run: |
+          helm install base charts/base --namespace instill-ai --create-namespace \
+            --set edition=k8s-ce:test \
+            --set apiGateway.image.tag=latest \
+            --set mgmtBackend.image.tag=latest \
+            --set console.image.tag=latest \
+            --set tags.observability=false
+
+      - name: Wait for base pods up
+        run: |
+          while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do
+            echo "$(kubectl get pods --namespace instill-ai)"
+            sleep 10
+          done
+
+      - name: Port-forward of api-gateway
+        run: |
+          API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o json | jq -r '.items[0].metadata.name')
+          kubectl --namespace instill-ai port-forward ${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 &
+          while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done
+
+      - name: Checkout repo (vdp)
+        uses: actions/checkout@v3
+        with:
+          repository: instill-ai/vdp
+
+      - name: Load .env file (vdp)
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+
+      - name: Launch Instill VDP (latest)
+        run: |
+          helm install vdp charts/vdp --namespace instill-ai --create-namespace \
+            --set edition=k8s-ce:test \
+            --set pipelineBackend.image.tag=latest \
+            --set connectorBackend.image.tag=latest \
+            --set controllerVDP.image.tag=latest \
+            --set tags.observability=false
+
+      - name: Wait for rollout vdp deployments
+        run: |
+         kubectl rollout status deployment vdp-connector-backend --namespace ${HELM_NAMESPACE}
+         kubectl rollout status deployment vdp-pipeline-backend --namespace ${HELM_NAMESPACE}
+         kubectl rollout status deployment vdp-controller-vdp --namespace ${HELM_NAMESPACE}
+         sleep 10
+
+      - name: Run ${{ inputs.component }} integration test (latest)
+        if: inputs.target == 'latest'
+        run: |
+          git clone https://github.com/instill-ai/${{ inputs.component }}.git
+          cd ${{ inputs.component }}
+          make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}
+
+  helm-integration-test-release-linux:
     if: inputs.target == 'release'
     runs-on: ubuntu-latest
     steps:
@@ -209,3 +327,127 @@ jobs:
           git clone -b v$COMPONENT_VERSION https://github.com/instill-ai/${{ inputs.component }}.git
           cd ${{ inputs.component }}
           make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}
+
+  helm-integration-test-release-mac:
+    if: inputs.target == 'release'
+    runs-on: [self-hosted,macOS,vdp]
+    steps:
+      - name: Set up environment
+        run: |
+          brew install make
+
+      - name: Checkout repo (vdp)
+        uses: actions/checkout@v3
+        with:
+          repository: instill-ai/vdp
+
+      - name: Load .env file (vdp)
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+
+      - name: Check if VDP Helm release exists
+        id: check-vdp-helm-release
+        run: |
+          if helm ls -n instill-ai | grep -q 'vdp'; then
+            echo "Helm release 'vdp' found."
+            echo "release_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "Helm release 'vdp' not found."
+          fi
+
+      - name: Uninstall VDP Helm Release
+        if: steps.check-vdp-helm-release.outputs.release_exists == 'true'
+        run: |
+          helm uninstall vdp --namespace instill-ai
+
+      - name: Install k6
+        run: |
+          brew install k6
+
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          repository: instill-ai/base
+
+      - name: Load .env file
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+
+      - name: Check if Base Helm release exists
+        id: check-base-helm-release
+        run: |
+          if helm ls -n instill-ai | grep -q 'base'; then
+            echo "Helm release 'base' found."
+            echo "release_exists=true" >> $GITHUB_OUTPUT
+          else
+            echo "Helm release 'base' not found."
+          fi
+
+      - name: Uninstall Base Helm Release
+        if: steps.check-base-helm-release.outputs.release_exists == 'true'
+        run: |
+          helm uninstall base --namespace instill-ai
+          kubectl delete namespace instill-ai
+
+      - name: Launch Helm Instill Base (release)
+        run: |
+          helm install base charts/base --namespace instill-ai --create-namespace \
+            --set edition=k8s-ce:test \
+            --set apiGateway.image.tag=${API_GATEWAY_VERSION} \
+            --set mgmtBackend.image.tag=${MGMT_BACKEND_VERSION} \
+            --set console.image.tag=${CONSOLE_VERSION} \
+            --set tags.observability=false
+
+      - name: Wait for base pods up
+        run: |
+          while [[ $(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o 'jsonpath={..status.phase}') != *"Running"* ]]; do
+            echo "$(kubectl get pods --namespace instill-ai)"
+            sleep 10
+          done
+
+      - name: Port-forward of base-api-gateway
+        run: |
+          API_GATEWAY_POD_NAME=$(kubectl get pods --namespace instill-ai -l "app.kubernetes.io/component=api-gateway,app.kubernetes.io/instance=base" -o json | jq -r '.items[0].metadata.name')
+          kubectl --namespace instill-ai port-forward ${API_GATEWAY_POD_NAME} ${API_GATEWAY_PORT}:${API_GATEWAY_PORT} > /dev/null 2>&1 &
+          while ! nc -vz localhost ${API_GATEWAY_PORT} > /dev/null 2>&1; do sleep 1; done
+
+      - name: Checkout repo (vdp)
+        uses: actions/checkout@v3
+        with:
+          repository: instill-ai/vdp
+
+      - name: Load .env file (vdp)
+        uses: cardinalby/export-env-action@v2
+        with:
+          envFile: .env
+
+      - name: Uppercase component name
+        id: uppercase
+        run: |
+          echo "COMPONENT_NAME=$(echo ${{ inputs.component }} | tr 'a-z-' 'A-Z_')" >> $GITHUB_OUTPUT
+
+      - name: Launch Helm Instill VDP (release)
+        run: |
+          helm install vdp charts/vdp --namespace instill-ai --create-namespace \
+            --set edition=k8s-ce:test \
+            --set pipelineBackend.image.tag=${PIPELINE_BACKEND_VERSION} \
+            --set connectorBackend.image.tag=${CONNECTOR_BACKEND_VERSION} \
+            --set controllerVDP.image.tag=${CONTROLLER_VDP_VERSION} \
+            --set tags.observability=false
+
+      - name: Wait for rollout vdp deployments
+        run: |
+         kubectl rollout status deployment vdp-connector-backend --namespace ${HELM_NAMESPACE}
+         kubectl rollout status deployment vdp-pipeline-backend --namespace ${HELM_NAMESPACE}
+         kubectl rollout status deployment vdp-controller-vdp --namespace ${HELM_NAMESPACE}
+         sleep 10
+
+      - name: Run ${{ inputs.component }} integration test (release)
+        env:
+          COMPONENT_VERSION: ${{ env[format('{0}_VERSION', steps.uppercase.outputs.COMPONENT_NAME)] }}
+        run: |
+          git clone -b v$COMPONENT_VERSION https://github.com/instill-ai/${{ inputs.component }}.git
+          cd ${{ inputs.component }}
+          make integration-test API_GATEWAY_URL=localhost:${API_GATEWAY_PORT}
diff --git a/.github/workflows/helm-integration-test-latest.yml b/.github/workflows/helm-integration-test-latest.yml
@@ -12,7 +12,7 @@ jobs:
       fail-fast: false
       matrix:
         component: [pipeline-backend, connector-backend, controller-vdp]
-    uses: instill-ai/vdp/.github/workflows/helm-integration-test-backend.yml@main
+    uses: instill-ai/vdp/.github/workflows/helm-integration-test-backend.yml@sarthakgupta/ins-1586-create-integration-tests-cicd-for-mac-in-base-model-and-vdp
     with:
       component: ${{ matrix.component }}
       target: latest
diff --git a/.github/workflows/helm-integration-test-release.yml b/.github/workflows/helm-integration-test-release.yml
@@ -2,14 +2,17 @@ name: Helm Integration Test (release)
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - sarthakgupta/ins-1586-create-integration-tests-cicd-for-mac-in-base-model-and-vdp
 
 jobs:
   backend:
     strategy:
       fail-fast: false
       matrix:
         component: [pipeline-backend, connector-backend, controller-vdp]
-    uses: instill-ai/vdp/.github/workflows/helm-integration-test-backend.yml@main
+    uses: instill-ai/vdp/.github/workflows/helm-integration-test-backend.yml@sarthakgupta/ins-1586-create-integration-tests-cicd-for-mac-in-base-model-and-vdp
     with:
       component: ${{ matrix.component }}
       target: release