From 01c3d1f7c6e6f36bb0215f711f0c7c123b978d03 Mon Sep 17 00:00:00 2001
From: huangzhangshu <109708205+JashBook@users.noreply.github.com>
Date: Fri, 8 Nov 2024 18:32:19 +0800
Subject: [PATCH] feat: cloud test on k3d (#479)

* chore: cloud test on k3d

* chore: add update oteld version
---
 .github/utils/manifests_version_update.sh |  18 ++++
 .github/workflows/cloud-test-k3d.yml      | 126 ++++++++++++++++++++++
 .github/workflows/test-cloud-k3d.yml      | 120 +++++++++++++++++++++
 3 files changed, 264 insertions(+)
 create mode 100644 .github/workflows/cloud-test-k3d.yml
 create mode 100644 .github/workflows/test-cloud-k3d.yml

diff --git a/.github/utils/manifests_version_update.sh b/.github/utils/manifests_version_update.sh
index e69e285..97aebf4 100644
--- a/.github/utils/manifests_version_update.sh
+++ b/.github/utils/manifests_version_update.sh
@@ -53,6 +53,19 @@ update_kbInstaller_version() {
     fi
 }
 
+update_oteld_version() {
+    oteld_version="${1}"
+    if [[ ! -f "${INSTALLER_CHART_FILE}" ]]; then
+        echo "$(tput -T xterm setaf 3)::warning title=Not found installer chart values file:${INSTALLER_CHART_FILE} $(tput -T xterm sgr0)"
+        return
+    fi
+    if [[ "$UNAME" == "Darwin" ]]; then
+        sed -i '' "s/^  oteld:.*/  oteld: \"${oteld_version}\"/" "${INSTALLER_CHART_FILE}"
+    else
+        sed -i "s/^  oteld:.*/  oteld: \"${oteld_version}\"/" "${INSTALLER_CHART_FILE}"
+    fi
+}
+
 update_manifests_file_version() {
     echo "MANIFESTS_FILE:${MANIFESTS_FILE}"
     if [[ ! -f "${MANIFESTS_FILE}" ]]; then
@@ -88,6 +101,11 @@ update_manifests_file_version() {
         for manifests_image in $(echo "${manifests_images}"); do
             for update_image in "${update_images[@]}"; do
                 if [[ "${manifests_image}" == "apecloud/${update_image}:"* ]]; then
+                    if [[ "${manifests_image}" == "apecloud/oteld:"* ]]; then
+                        oteld_version="${manifests_image#*:}"
+                        update_oteld_version "${oteld_version}"
+                    fi
+
                     if [[ "${manifests_image}" == "apecloud/kubeblocks-installer:"* ]]; then
                         kbInstaller_version="${RELEASE_VERSION}-${KUBEBLOCKS_VERSION}-offline"
                         yq e -i ".${chart_name}[0].images[${image_num}]=\"apecloud/${update_image}:${kbInstaller_version}\"" "${MANIFESTS_FILE}"
diff --git a/.github/workflows/cloud-test-k3d.yml b/.github/workflows/cloud-test-k3d.yml
new file mode 100644
index 0000000..553f5eb
--- /dev/null
+++ b/.github/workflows/cloud-test-k3d.yml
@@ -0,0 +1,126 @@
+name: Cloud E2E Test on K3D
+
+on:
+  workflow_call:
+    inputs:
+      APECLOUD_VERSION:
+        description: 'ApeCloud release version'
+        type: string
+        required: false
+        default: 'v0.3.5'
+      KUBEBLOCKS_VERSION:
+        description: 'kubeblocks release version'
+        type: string
+        required: false
+        default: 'latest'
+      K3S_VERSION:
+        description: 'k8s cluster version (e.g. 1.28)'
+        type: string
+        required: false
+        default: '1.28'
+      E2ETEST_BRANCH:
+        description: 'e2etest branch name'
+        type: string
+        required: false
+        default: 'main'
+      APECD_REF:
+        description: "The branch name of apecloud-cd"
+        type: string
+        required: false
+        default: 'main'
+      ARGS:
+        description: "Test args"
+        type: string
+        required: false
+        default: ''
+  workflow_dispatch:
+    inputs:
+      TEST_TYPE:
+        description: 'test type (e.g.  wesql|postgresql|redis|mongodb)'
+        type: string
+        required: false
+        default: ''
+      APECLOUD_VERSION:
+        description: 'ApeCloud release version'
+        type: string
+        required: false
+        default: 'v0.3.5'
+      KUBEBLOCKS_VERSION:
+        description: 'kubeblocks release version'
+        type: string
+        required: false
+        default: 'latest'
+      K3S_VERSION:
+        description: 'k8s cluster version (e.g. 1.28)'
+        type: string
+        required: false
+        default: '1.28'
+      E2ETEST_BRANCH:
+        description: 'e2etest branch name'
+        type: string
+        required: false
+        default: 'main'
+      APECD_REF:
+        description: "The branch name of apecloud-cd"
+        type: string
+        required: false
+        default: 'main'
+      ARGS:
+        description: "Test args"
+        type: string
+        required: false
+        default: ''
+
+run-name: ApeCloud:${{ inputs.APECLOUD_VERSION }} K3S:${{ inputs.K3S_VERSION }}
+
+env:
+  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  e2e-test:
+    uses: ./.github/workflows/test-cloud-k3d.yml
+    with:
+      release-version: ${{ inputs.APECLOUD_VERSION }}
+      kubeblocks-version: ${{ inputs.KUBEBLOCKS_VERSION }}
+      test-type: ${{ inputs.TEST_TYPE }}
+      test-type-name: "e2e-test"
+      test-args: ${{ inputs.ARGS }}
+      k3s-version: ${{ inputs.K3S_VERSION }}
+      e2etest-branch: ${{ inputs.E2ETEST_BRANCH }}
+    secrets: inherit
+
+  send-message:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs: [ e2e-test ]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: apecloud/apecloud-cd
+          path: ./
+          ref: ${{ inputs.APECD_REF }}
+
+      - name: send message
+        id: get_trigger_mode
+        run: |
+          
+          TEST_RESULT="deploy-cloud|${{ needs.e2e-test.outputs.deploy-result }}"
+          echo "$TEST_RESULT"
+          TEST_RESULT="${TEST_RESULT}##${{ needs.e2e-test.outputs.test-type }}|${{ needs.e2e-test.outputs.test-result }}"
+          echo "$TEST_RESULT"
+          TEST_RESULT=$( bash .github/utils/utils.sh --type 12 \
+              --github-repo "${{ github.repository }}" \
+              --github-token "${{ env.GITHUB_TOKEN }}" \
+              --test-result "${TEST_RESULT}" \
+              --run-id "$GITHUB_RUN_ID" )
+          
+          date_ret=$(date +%Y-%m-%d-%T)
+          test_title="[${{ inputs.APECLOUD_VERSION }}] Test ApeCloud on K3S:${{ inputs.K3S_VERSION }} [${date_ret}]"
+          python3 .github/utils/send_mesage.py \
+              --url "${{ vars.TEST_BOT_WEBHOOK }}" \
+              --title "$test_title" \
+              --result "$TEST_RESULT"
diff --git a/.github/workflows/test-cloud-k3d.yml b/.github/workflows/test-cloud-k3d.yml
new file mode 100644
index 0000000..a1766da
--- /dev/null
+++ b/.github/workflows/test-cloud-k3d.yml
@@ -0,0 +1,120 @@
+name: Test Cloud On K3D
+
+on:
+  workflow_dispatch:
+    inputs:
+      cloud-version:
+        description: 'kb-cloud version (e.g. 0.28.50)'
+        type: string
+        required: false
+        default: '0.28.50'
+      k3s-version:
+        description: 'k3s cluster version (e.g. 1.30)'
+        type: string
+        required: false
+        default: '1.30'
+  workflow_call:
+    inputs:
+      cloud-version:
+        description: 'kb-cloud version (e.g. 0.28.50)'
+        type: string
+        required: false
+        default: '0.28.50'
+      k3s-version:
+        description: 'k3s cluster version (e.g. 1.30)'
+        type: string
+        required: false
+        default: '1.30'
+    outputs:
+      test-result:
+        description: "e2e test result"
+        value: "${{ jobs.cloud-test-k3d.outputs.test-result }}"
+
+
+env:
+  GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  K3D_NAME: kbcloud
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  cloud-test-k3d:
+    name: run ${{ inputs.test-type-name }}
+    runs-on: ubuntu-latest
+    outputs:
+      deploy-result: ${{ steps.deploy_cloud.outputs.deploy-result }}
+      test-result: ${{ steps.test_result.outputs.test-result }}
+      test-type: ${{ steps.test_result.outputs.test-type }}
+    steps:
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v3
+        with:
+          version: "v1.30.4"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: "v3.15.4"
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.22"
+
+      - name: setup k3d k3s
+        uses: nolar/setup-k3d-k3s@v1
+        with:
+          k3d-name: "${{ env.K3D_NAME }}"
+          version: "v${{ inputs.k3s-version }}"
+          github-token: ${{ env.GITHUB_TOKEN }}
+          k3d-args: -p 443:443@loadbalancer -p 80:80@loadbalancer --agents 2
+
+      - name: get kubeconfig
+        run: |
+          K3S_CLUSTE_NAME=$(kubectl config get-clusters|(grep -v NAME||true))
+          kubectl config set-cluster ${K3S_CLUSTE_NAME} --insecure-skip-tls-verify=true
+          K3D_KUBECONFIG=$(kubectl config view --minify -o yaml --raw)
+          kubeconfig_file_name="k3d_kubeconfig_file.yaml"
+          touch ${kubeconfig_file_name}
+          echo "${K3D_KUBECONFIG}" > ${kubeconfig_file_name}
+          yq e -i '.clusters[0].cluster.server = "https://kubernetes.default:443"' ${kubeconfig_file_name}
+          K3D_KUBECONFIG=$(cat ${kubeconfig_file_name})
+          echo "${K3D_KUBECONFIG}"
+          echo K3D_KUBECONFIG="${K3D_KUBECONFIG}" >> $GITHUB_ENV
+
+      - name: install cert-manager
+        run: |
+          helm repo add jetstack https://charts.jetstack.io --force-update
+          helm install \
+            cert-manager jetstack/cert-manager \
+            --namespace cert-manager \
+            --create-namespace \
+            --version v1.16.0 \
+            --set crds.enabled=true
+
+      - name: install kb cloud
+        run: |
+          helm repo add kb-chart https://apecloud.github.io/helm-charts
+          
+          helm install installer kb-chart/kb-cloud-installer \
+            --version ${{ inputs.cloud-version }} \
+            --set version=${{ inputs.cloud-version }} \
+            --set kubernetesProvider=k3d \
+            --set fqdn.domain=mytest.kubeblocks.com \
+            --set tls.enabled=true
+
+      - name: check kb-cloud-installer pod Completed
+        run: |
+          for i in {1..360}; do
+            if kubectl get pods | grep "kb-cloud-installer" | grep Completed ; then
+              kubectl get pods -n kb-cloud | grep Running
+              echo "KB Cloud is ready"
+              break
+            fi
+            echo "Waiting for KB Cloud to be ready..."
+            kubectl get pods -n kb-cloud
+            echo ""
+            sleep 5
+          done