CLIP-1878: Update KinD scripts and config to be able to run DC functi…

…onal tests (#812) * Update kind scripts and config to run functional tests * Increase readiness probe failure threshold * Remove debug selector --------- Co-authored-by: Yevhen Ivantsov <[email protected]>
atlassian · Jun 4, 2024 · 857ba11 · 857ba11
1 parent cf6204f
commit 857ba11
Show file tree

Hide file tree

Showing 9 changed files with 303 additions and 51 deletions.
diff --git a/.github/workflows/kind.yaml b/.github/workflows/kind.yaml
@@ -22,9 +22,9 @@ jobs:
     runs-on: ubuntu-latest
     env:
       # See: https://github.com/kubernetes-sigs/kind/tags
-      KIND_VERSION: "v0.20.0"
+      KIND_VERSION: "v0.23.0"
       # See: https://hub.docker.com/r/kindest/node/tags
-      K8S_VERSION: "v1.28.0"
+      K8S_VERSION: "v1.30.0"
       DC_APP: ${{inputs.dc_app}}
       LICENSE: ${{ secrets[format('{0}_LICENSE', inputs.dc_app)] }}
 

diff --git a/src/test/config/kind/backdoor-svc.yaml b/src/test/config/kind/backdoor-svc.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: DC_APP_REPLACEME-0
+  namespace: atlassian
+spec:
+  type: NodePort
+  selector:
+    statefulset.kubernetes.io/pod-name: DC_APP_REPLACEME-0
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      protocol: TCP
+      nodePort: 30008
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: DC_APP_REPLACEME-1
+  namespace: atlassian
+spec:
+  type: NodePort
+  selector:
+    statefulset.kubernetes.io/pod-name: DC_APP_REPLACEME-1
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
+      protocol: TCP
+      nodePort: 30009
diff --git a/src/test/config/kind/common-values.yaml b/src/test/config/kind/common-values.yaml
@@ -1,19 +1,22 @@
 DC_APP_REPLACEME:
   # claim little resources as there are limitations in GitHub actions
+  # we're not setting any limits, so containers will use as much resources as necessary
   resources:
     container:
       requests:
         cpu: 20m
         memory: 1G
   # we want to test scaling to 2 and making sure nodes have joined the cluster
+  # this is also required for DC functional tests
   clustering:
     enabled: true
 
   # this works for Bitbucket and Bamboo only which support unattended setup
   sysadminCredentials:
     secretName: DC_APP_REPLACEME-admin
-  displayName: DC_APP_REPLACEME
+  displayName: DISPLAY_NAME
 
+  # license secret will be ignored in Jira
   license:
     secretName: DC_APP_REPLACEME-app-license
     secretKey: license
@@ -25,6 +28,44 @@ DC_APP_REPLACEME:
   additionalCertificates:
     secretName: certificate
 
+  # Bitbucket Only
+  sshService:
+    enabled: true
+    type: NodePort
+
+  # 2222 is a mapped Bitbucket SSH port
+  additionalEnvironmentVariables:
+    - name: PLUGIN_SSH_BASEURL
+      value: ssh://bitbucket.172.17.0.1.nip.io:2222
+
+  # non existing props will be ignored, so it's safe to set them for all products
+  # these extra sys properties are mostly for the functional tests
+  additionalJvmArgs:
+    - -Dupm.plugin.upload.enabled=true
+    - -Dfeature.getting.started.page=false
+    - -Datlassian.darkfeature.jira.onboarding.feature.disabled=true
+    - -Djira.websudo.is.disabled=true
+    - -Datlassian.allow.insecure.url.parameter.login=true
+
+  # Jira func tests will setup Jira, and during this time status will report 500
+  # to avoid marking container as non-ready, let's bump failureThreshold for a readinessProbe
+  readinessProbe:
+    failureThreshold: 100
+
+  # Bitbucket functional tests expect 3 Mesh nodes,
+  # we're enabling auto-registration jobs to make sure Mesh nodes register with the server
+  mesh:
+    enabled: true
+    nodeAutoRegistration: true
+    setByDefault: true
+    resources:
+      container:
+        requests:
+          cpu: "10m"
+          memory: "1G"
+    image:
+      tag: "latest"
+
 database:
   type: DB_TYPE_REPLACEME
   url: jdbc:postgresql://postgres:5432/DC_APP_REPLACEME
@@ -47,6 +88,7 @@ volumes:
       # this is the default storageclass name created when deploying the provisioner
       storageClassName: nfs-client
 
+# just testing templating annotations
 podAnnotations:
   annotation: "{{ \"podOfTucuxis\" | upper }}"
   quote: "true"
@@ -58,6 +100,9 @@ ingress:
   create: true
   host: localhost
   https: false
+  proxyConnectTimeout: 300
+  proxyReadTimeout: 300
+  proxySendTimeout: 300
 
 monitoring:
   exposeJmxMetrics: true
@@ -75,3 +120,6 @@ synchrony:
         memory: 1G
   additionalCertificates:
     secretName: certificate
+
+opensearch:
+  install: true
diff --git a/src/test/config/kind/hostpath-pv.yaml b/src/test/config/kind/hostpath-pv.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: hostpath-shared-home
+  namespace: atlassian
+spec:
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Delete
+  hostPath:
+    path: /mnt/data
+    type: DirectoryOrCreate
+  claimRef:
+    namespace: atlassian
+    name: hostpath-shared-home-pvc
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hostpath-shared-home-pvc
+  namespace: atlassian
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+
diff --git a/src/test/config/kind/kind-config.yml b/src/test/config/kind/kind-config.yml
@@ -22,3 +22,27 @@ nodes:
   - containerPort: 443
     hostPort: 443
     protocol: TCP
+  # map registry NodePort to host port to be able to
+  # build and push images to an internal registry
+  - containerPort: 32767
+    hostPort: 5001
+    protocol: TCP
+  # this is required for Bitbucket tests to map ssh NodePort to 2222 hostPort
+  - containerPort: 30007
+    hostPort: 2222
+    protocol: TCP
+  # these are so called backdoor services that are used
+  # in dc functional tests when nodes have to be directly accessed
+  - containerPort: 30008
+    hostPort: 81
+    protocol: TCP
+  - containerPort: 30009
+    hostPort: 82
+    protocol: TCP
+containerdConfigPatches:
+# configure container runtime to trust an insecure registry
+- |-
+    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."172.17.0.1:5001"]
+      endpoint = ["http://172.17.0.1:5001"]
+    [plugins."io.containerd.grpc.v1.cri".registry.configs."172.17.0.1:5001".tls]
+      insecure_skip_verify = true
diff --git a/src/test/config/kind/registry.yaml b/src/test/config/kind/registry.yaml
@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: registry
+  namespace: atlassian
+spec:
+  type: NodePort
+  selector:
+    app: registry
+  ports:
+    - name: registry-tcp
+      protocol: TCP
+      port: 5000
+      targetPort: 5000
+      nodePort: 32767
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: registry
+  name: registry
+  namespace: atlassian
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: registry
+  template:
+    metadata:
+      labels:
+        app: registry
+    spec:
+      containers:
+      - name: registry
+        image: registry:2
+        ports:
+        - containerPort: 5000
diff --git a/src/test/scripts/kind/configure_kind.sh b/src/test/scripts/kind/configure_kind.sh
@@ -7,31 +7,43 @@ echo "[INFO]: current-context:" $(kubectl config current-context)
 echo "[INFO]: environment-kubeconfig:" "${KUBECONFIG}"
 
 kubectl create namespace atlassian
-echo "[INFO]: Deploy Nginx ingress controller"
 
+# even though there's a kind command to load a local image directly to KinD container runtime
+# let's deploy an insecure registry in case we need it for any further tests
+echo "[INFO]: Deploy ephemeral container registry"
+kubectl apply -f src/test/config/kind/registry.yaml
+
+echo "[INFO]: Deploy Nginx ingress controller"
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
 kubectl wait --for=condition=ready pod \
         --selector=app.kubernetes.io/component=controller \
-        --timeout=90s \
+        --timeout=180s \
         -n ingress-nginx
 
-echo "[INFO]: Deploy NFS server"
-helm install nfs src/test/infrastructure/nfs-server \
-    --set image.tag=2.0 \
-     -n atlassian \
-     --timeout=360s \
-     --wait
-
-nfs_server_ip=$(kubectl get svc/nfs-nfs-server -n atlassian -o jsonpath='{.spec.clusterIP}')
-
-echo "[INFO]: Deploy NFS volume provisioner. Using ${nfs_server_ip} as NFS server IP"
-helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner
-helm repo update
-
-helm install nfs-volume-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
-     -f src/test/config/kind/nfs-values.yaml \
-     --set nfs.server=${nfs_server_ip} \
-     --set nfs.path=/srv/nfs \
-     -n atlassian \
-     --timeout=360s \
-     --wait
+# this is for local runs, because existing nfs server images does not run on arm64 platforms
+# instead, we create a hostPath RWX volume and override the default common settings
+if [ -z "${HOSTPATH_PV}" ]; then
+  echo "[INFO]: Deploy NFS server"
+  helm install nfs src/test/infrastructure/nfs-server \
+      --set image.tag=2.0 \
+       -n atlassian \
+       --timeout=360s \
+       --wait
+
+  nfs_server_ip=$(kubectl get svc/nfs-nfs-server -n atlassian -o jsonpath='{.spec.clusterIP}')
+
+  echo "[INFO]: Deploy NFS volume provisioner. Using ${nfs_server_ip} as NFS server IP"
+  helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner
+  helm repo update
+
+  helm install nfs-volume-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
+       -f src/test/config/kind/nfs-values.yaml \
+       --set nfs.server=${nfs_server_ip} \
+       --set nfs.path=/srv/nfs \
+       -n atlassian \
+       --timeout=360s \
+       --wait
+else
+  echo "[INFO]: Creating a hostPath PersistentVolume"
+  kubectl apply -f src/test/config/kind/hostpath-pv.yaml
+fi
diff --git a/src/test/scripts/kind/create_kind.sh b/src/test/scripts/kind/create_kind.sh
@@ -2,9 +2,18 @@
 
 set -e
 
-curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64"
-chmod +x ./kind
-sudo mv ./kind /usr/local/bin/kind
+# set defaults, in case env vars aren't exported
+export K8S_VERSION="${K8S_VERSION:-v1.30.0}"
+export KIND_VERSION="${KIND_VERSION:-v0.23.0}"
+
+if [ -z "${SKIP_DOWNLOAD_KIND}" ]; then
+  echo "[INFO]: Downloading KinD ${KIND_VERSION}"
+  curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64"
+  chmod +x ./kind
+  sudo mv ./kind /usr/local/bin/kind
+fi
+
+echo "[INFO]: Using config file src/test/config/kind/kind-config.yml"
 
 kind create cluster --name=atl-kind \
                     --image=kindest/node:${K8S_VERSION} \