Skip to content

Commit

Permalink
feat: cloud test on k3d (#479)
Browse files Browse the repository at this point in the history 
* chore: cloud test on k3d

* chore: add update oteld version
  • Loading branch information
@JashBook
JashBook authored Nov 8, 2024
1 parent 64647df commit 01c3d1f
Show file tree
Hide file tree
Showing 3 changed files with 264 additions and 0 deletions.
18 changes: 18 additions & 0 deletions .github/utils/manifests_version_update.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,19 @@ update_kbInstaller_version() {
fi
}

update_oteld_version() {
oteld_version="${1}"
if [[ ! -f "${INSTALLER_CHART_FILE}" ]]; then
echo "$(tput -T xterm setaf 3)::warning title=Not found installer chart values file:${INSTALLER_CHART_FILE} $(tput -T xterm sgr0)"
return
fi
if [[ "$UNAME" == "Darwin" ]]; then
sed -i '' "s/^ oteld:.*/ oteld: \"${oteld_version}\"/" "${INSTALLER_CHART_FILE}"
else
sed -i "s/^ oteld:.*/ oteld: \"${oteld_version}\"/" "${INSTALLER_CHART_FILE}"
fi
}

update_manifests_file_version() {
echo "MANIFESTS_FILE:${MANIFESTS_FILE}"
if [[ ! -f "${MANIFESTS_FILE}" ]]; then
Expand Down Expand Up @@ -88,6 +101,11 @@ update_manifests_file_version() {
for manifests_image in $(echo "${manifests_images}"); do
for update_image in "${update_images[@]}"; do
if [[ "${manifests_image}" == "apecloud/${update_image}:"* ]]; then
if [[ "${manifests_image}" == "apecloud/oteld:"* ]]; then
oteld_version="${manifests_image#*:}"
update_oteld_version "${oteld_version}"
fi

if [[ "${manifests_image}" == "apecloud/kubeblocks-installer:"* ]]; then
kbInstaller_version="${RELEASE_VERSION}-${KUBEBLOCKS_VERSION}-offline"
yq e -i ".${chart_name}[0].images[${image_num}]=\"apecloud/${update_image}:${kbInstaller_version}\"" "${MANIFESTS_FILE}"
Expand Down
126 changes: 126 additions & 0 deletions .github/workflows/cloud-test-k3d.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
name: Cloud E2E Test on K3D

on:
workflow_call:
inputs:
APECLOUD_VERSION:
description: 'ApeCloud release version'
type: string
required: false
default: 'v0.3.5'
KUBEBLOCKS_VERSION:
description: 'kubeblocks release version'
type: string
required: false
default: 'latest'
K3S_VERSION:
description: 'k8s cluster version (e.g. 1.28)'
type: string
required: false
default: '1.28'
E2ETEST_BRANCH:
description: 'e2etest branch name'
type: string
required: false
default: 'main'
APECD_REF:
description: "The branch name of apecloud-cd"
type: string
required: false
default: 'main'
ARGS:
description: "Test args"
type: string
required: false
default: ''
workflow_dispatch:
inputs:
TEST_TYPE:
description: 'test type (e.g. wesql|postgresql|redis|mongodb)'
type: string
required: false
default: ''
APECLOUD_VERSION:
description: 'ApeCloud release version'
type: string
required: false
default: 'v0.3.5'
KUBEBLOCKS_VERSION:
description: 'kubeblocks release version'
type: string
required: false
default: 'latest'
K3S_VERSION:
description: 'k8s cluster version (e.g. 1.28)'
type: string
required: false
default: '1.28'
E2ETEST_BRANCH:
description: 'e2etest branch name'
type: string
required: false
default: 'main'
APECD_REF:
description: "The branch name of apecloud-cd"
type: string
required: false
default: 'main'
ARGS:
description: "Test args"
type: string
required: false
default: ''

run-name: ApeCloud:${{ inputs.APECLOUD_VERSION }} K3S:${{ inputs.K3S_VERSION }}

env:
GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}

permissions:
id-token: write
contents: read

jobs:
e2e-test:
uses: ./.github/workflows/test-cloud-k3d.yml
with:
release-version: ${{ inputs.APECLOUD_VERSION }}
kubeblocks-version: ${{ inputs.KUBEBLOCKS_VERSION }}
test-type: ${{ inputs.TEST_TYPE }}
test-type-name: "e2e-test"
test-args: ${{ inputs.ARGS }}
k3s-version: ${{ inputs.K3S_VERSION }}
e2etest-branch: ${{ inputs.E2ETEST_BRANCH }}
secrets: inherit

send-message:
if: ${{ always() }}
runs-on: ubuntu-latest
needs: [ e2e-test ]
steps:
- uses: actions/checkout@v4
with:
repository: apecloud/apecloud-cd
path: ./
ref: ${{ inputs.APECD_REF }}

- name: send message
id: get_trigger_mode
run: |
TEST_RESULT="deploy-cloud|${{ needs.e2e-test.outputs.deploy-result }}"
echo "$TEST_RESULT"
TEST_RESULT="${TEST_RESULT}##${{ needs.e2e-test.outputs.test-type }}|${{ needs.e2e-test.outputs.test-result }}"
echo "$TEST_RESULT"
TEST_RESULT=$( bash .github/utils/utils.sh --type 12 \
--github-repo "${{ github.repository }}" \
--github-token "${{ env.GITHUB_TOKEN }}" \
--test-result "${TEST_RESULT}" \
--run-id "$GITHUB_RUN_ID" )
date_ret=$(date +%Y-%m-%d-%T)
test_title="[${{ inputs.APECLOUD_VERSION }}] Test ApeCloud on K3S:${{ inputs.K3S_VERSION }} [${date_ret}]"
python3 .github/utils/send_mesage.py \
--url "${{ vars.TEST_BOT_WEBHOOK }}" \
--title "$test_title" \
--result "$TEST_RESULT"
120 changes: 120 additions & 0 deletions .github/workflows/test-cloud-k3d.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
name: Test Cloud On K3D

on:
workflow_dispatch:
inputs:
cloud-version:
description: 'kb-cloud version (e.g. 0.28.50)'
type: string
required: false
default: '0.28.50'
k3s-version:
description: 'k3s cluster version (e.g. 1.30)'
type: string
required: false
default: '1.30'
workflow_call:
inputs:
cloud-version:
description: 'kb-cloud version (e.g. 0.28.50)'
type: string
required: false
default: '0.28.50'
k3s-version:
description: 'k3s cluster version (e.g. 1.30)'
type: string
required: false
default: '1.30'
outputs:
test-result:
description: "e2e test result"
value: "${{ jobs.cloud-test-k3d.outputs.test-result }}"


env:
GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
K3D_NAME: kbcloud

permissions:
id-token: write
contents: read

jobs:
cloud-test-k3d:
name: run ${{ inputs.test-type-name }}
runs-on: ubuntu-latest
outputs:
deploy-result: ${{ steps.deploy_cloud.outputs.deploy-result }}
test-result: ${{ steps.test_result.outputs.test-result }}
test-type: ${{ steps.test_result.outputs.test-type }}
steps:
- name: Setup kubectl
uses: azure/setup-kubectl@v3
with:
version: "v1.30.4"

- name: Install Helm
uses: azure/setup-helm@v3
with:
version: "v3.15.4"

- name: Setup Go
uses: actions/setup-go@v3
with:
go-version: "1.22"

- name: setup k3d k3s
uses: nolar/setup-k3d-k3s@v1
with:
k3d-name: "${{ env.K3D_NAME }}"
version: "v${{ inputs.k3s-version }}"
github-token: ${{ env.GITHUB_TOKEN }}
k3d-args: -p 443:443@loadbalancer -p 80:80@loadbalancer --agents 2

- name: get kubeconfig
run: |
K3S_CLUSTE_NAME=$(kubectl config get-clusters|(grep -v NAME||true))
kubectl config set-cluster ${K3S_CLUSTE_NAME} --insecure-skip-tls-verify=true
K3D_KUBECONFIG=$(kubectl config view --minify -o yaml --raw)
kubeconfig_file_name="k3d_kubeconfig_file.yaml"
touch ${kubeconfig_file_name}
echo "${K3D_KUBECONFIG}" > ${kubeconfig_file_name}
yq e -i '.clusters[0].cluster.server = "https://kubernetes.default:443"' ${kubeconfig_file_name}
K3D_KUBECONFIG=$(cat ${kubeconfig_file_name})
echo "${K3D_KUBECONFIG}"
echo K3D_KUBECONFIG="${K3D_KUBECONFIG}" >> $GITHUB_ENV
- name: install cert-manager
run: |
helm repo add jetstack https://charts.jetstack.io --force-update
helm install \
cert-manager jetstack/cert-manager \
--namespace cert-manager \
--create-namespace \
--version v1.16.0 \
--set crds.enabled=true
- name: install kb cloud
run: |
helm repo add kb-chart https://apecloud.github.io/helm-charts
helm install installer kb-chart/kb-cloud-installer \
--version ${{ inputs.cloud-version }} \
--set version=${{ inputs.cloud-version }} \
--set kubernetesProvider=k3d \
--set fqdn.domain=mytest.kubeblocks.com \
--set tls.enabled=true
- name: check kb-cloud-installer pod Completed
run: |
for i in {1..360}; do
if kubectl get pods | grep "kb-cloud-installer" | grep Completed ; then
kubectl get pods -n kb-cloud | grep Running
echo "KB Cloud is ready"
break
fi
echo "Waiting for KB Cloud to be ready..."
kubectl get pods -n kb-cloud
echo ""
sleep 5
done

0 comments on commit 01c3d1f

Please sign in to comment.