Enrich pod health check

Project-HAMi · Jan 10, 2025 · 5edd989 · 5edd989
1 parent 72621e8
commit 5edd989
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 2 deletions.
diff --git a/hack/deploy-helm.sh b/hack/deploy-helm.sh
@@ -63,3 +63,9 @@ util::exec_cmd helm --debug upgrade --install --create-namespace --cleanup-on-fa
              --set devicePlugin.passDeviceSpecsEnabled=false \
              --version "${HELM_VER}" --wait --timeout 20m   --kubeconfig "${KUBE_CONF}"
 
+# check pod running status
+kubectl  --kubeconfig "${KUBE_CONF}" get po  -n "${TARGET_NS}"
+
+if ! util:check_pods_status "${KUBE_CONF}" "${TARGET_NS}" ; then
+  return 1
+fi
diff --git a/hack/util.sh b/hack/util.sh
@@ -86,3 +86,32 @@ function util::wait_ip_reachable(){
       fi
     done
 }
+
+# checking pods in namespace works
+function check_pods_status() {
+  local kubeconfig=${1:-""}
+  local namespace=${2:-"hami-system"}
+
+  local unhealthy_pods
+  unhealthy_pods=$(kubectl get po -n "$namespace" --kubeconfig "$kubeconfig" | grep -Ev "^(NAME|.*Running.*|.*Succeeded.*)")
+
+  if [[ -n "$unhealthy_pods" ]]; then
+    echo "Found unhealthy_pods pods in namespace $namespace:"
+    echo "$unhealthy_pods"
+
+    for pod in $unhealthy_pods; do
+      echo "Describing pod: $pod"
+      kubectl describe po "$pod" -n "$namespace" --kubeconfig "$kubeconfig"
+
+      echo "Fetching logs for pod: $pod"
+      kubectl logs "$pod" -n "$namespace" --kubeconfig "$kubeconfig"
+      echo "---------------------------------------------------"
+    done
+
+    return 1
+  else
+
+    echo "PASS: All Pods are in Running state."
+    return 0
+  fi
+}
diff --git a/test/e2e/pod/test_pod.go b/test/e2e/pod/test_pod.go
@@ -63,7 +63,7 @@ var _ = ginkgo.Describe("Pod E2E Tests", ginkgo.Ordered, func() {
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	})
 
-	ginkgo.It("create single pod with configuration", func() {
+	ginkgo.It("create single pod with CUDA configuration", func() {
 		newPod = utils.Pod.DeepCopy()
 		newPod.Name = newPod.Name + utils.GetRandom()
 
@@ -92,7 +92,6 @@ var _ = ginkgo.Describe("Pod E2E Tests", ginkgo.Ordered, func() {
 		newPod.Name = newPod.Name + utils.GetRandom()
 		newPod.Spec.Containers = append(newPod.Spec.Containers, newPod.Spec.Containers[0])
 		newPod.Spec.Containers = append(newPod.Spec.Containers, newPod.Spec.Containers[0])
-		//newPod.Spec.Containers[0].Name = newPod.Spec.Containers[0].Name + utils.GetRandom()
 		newPod.Spec.Containers[1].Name = newPod.Spec.Containers[0].Name + utils.GetRandom()
 		newPod.Spec.Containers[2].Name = newPod.Spec.Containers[0].Name + utils.GetRandom()