refactor(statefulset): extend statefulset check logic

robolaunch · Mar 7, 2024 · 7d19554 · 7d19554
1 parent f97bd01
commit 7d19554
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 14 deletions.
diff --git a/pkg/controllers/v1alpha2/production/ros2_workload/check.go b/pkg/controllers/v1alpha2/production/ros2_workload/check.go
@@ -96,7 +96,7 @@ func (r *ROS2WorkloadReconciler) reconcileCheckStatefulSets(ctx context.Context,
 		volumesReady = volumesReady && pvcStatus.Resource.Created
 	}
 
-	if instance.Status.DiscoveryServerStatus.Status.ConfigMapStatus.Created && volumesReady {
+	if instance.Status.DiscoveryServerStatus.Status.ConfigMapStatus.Created && volumesReady && len(instance.Spec.LaunchContainers) == len(instance.Status.StatefulSetStatuses) {
 		for key, ssStatus := range instance.Status.StatefulSetStatuses {
 
 			ssQuery := &appsv1.StatefulSet{}
@@ -109,6 +109,10 @@ func (r *ROS2WorkloadReconciler) reconcileCheckStatefulSets(ctx context.Context,
 				// check if there are any inconsistencies
 				launchContainer := instance.Spec.LaunchContainers[key]
 				actualContainer := ssQuery.Spec.Template.Spec.Containers[0]
+				actualContainerStatus := corev1.ContainerStatus{}
+				if len(ssStatus.ContainerStatuses) > 0 {
+					actualContainerStatus = ssStatus.ContainerStatuses[0]
+				}
 
 				volumeMountsSynced := true
 				for _, vmDesired := range launchContainer.Container.VolumeMounts {
@@ -121,21 +125,41 @@ func (r *ROS2WorkloadReconciler) reconcileCheckStatefulSets(ctx context.Context,
 					volumeMountsSynced = volumeMountsSynced && volumePresent
 				}
 
+				envSynced := true
+				for _, envDesired := range launchContainer.Container.Env {
+					envPresent := false
+					for _, envActual := range actualContainer.Env {
+						if reflect.DeepEqual(envDesired, envActual) {
+							envPresent = true
+						}
+					}
+					envSynced = envSynced && envPresent
+				}
+
 				// changes that needs resource updates
 				if !reflect.DeepEqual(launchContainer.Replicas, ssQuery.Spec.Replicas) ||
 					!reflect.DeepEqual(launchContainer.Container.Image, actualContainer.Image) ||
 					!reflect.DeepEqual(launchContainer.Container.Command, actualContainer.Command) ||
 					!reflect.DeepEqual(launchContainer.Container.Resources, actualContainer.Resources) ||
-					!reflect.DeepEqual(launchContainer.Container.SecurityContext, actualContainer.SecurityContext) {
+					!reflect.DeepEqual(launchContainer.Container.SecurityContext, actualContainer.SecurityContext) ||
+					!envSynced {
 
-					err = r.updateStatefulSet(ctx, instance, key)
-					if err != nil {
-						return err
+					if !actualContainerStatus.Ready {
+						err = r.Delete(ctx, ssQuery)
+						if err != nil {
+							return err
+						}
+
+						ssStatus.Resource.Created = false
+						ssStatus.Status = appsv1.StatefulSetStatus{}
+						ssStatus.ContainerStatuses = []corev1.ContainerStatus{}
+					} else {
+						err = r.updateStatefulSet(ctx, instance, key)
+						if err != nil {
+							return err
+						}
 					}
 
-					ssStatus.Resource.Created = false
-					ssStatus.Status = appsv1.StatefulSetStatus{}
-					ssStatus.ContainerStatuses = []corev1.ContainerStatus{}
 					continue
 				}
 

diff --git a/pkg/controllers/v1alpha2/production/ros2_workload/register.go b/pkg/controllers/v1alpha2/production/ros2_workload/register.go
@@ -1,11 +1,14 @@
 package ros2_workload
 
 import (
+	"context"
+
 	robotv1alpha2 "github.com/robolaunch/robot-operator/pkg/api/roboscale.io/v1alpha2"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 )
 
-func (r *ROS2WorkloadReconciler) registerPVCs(instance *robotv1alpha2.ROS2Workload) {
+func (r *ROS2WorkloadReconciler) registerPVCs(ctx context.Context, instance *robotv1alpha2.ROS2Workload) error {
 
 	pvcStatuses := []robotv1alpha2.OwnedPVCStatus{}
 
@@ -23,13 +26,31 @@ func (r *ROS2WorkloadReconciler) registerPVCs(instance *robotv1alpha2.ROS2Worklo
 		}
 		instance.Status.PVCStatuses = pvcStatuses
 	}
+
+	return nil
 }
 
-func (r *ROS2WorkloadReconciler) registerStatefulSets(instance *robotv1alpha2.ROS2Workload) {
+func (r *ROS2WorkloadReconciler) registerStatefulSets(ctx context.Context, instance *robotv1alpha2.ROS2Workload) error {
 
 	ssStatuses := []robotv1alpha2.OwnedStatefulSetStatus{}
 
 	if len(instance.Spec.LaunchContainers) != len(instance.Status.StatefulSetStatuses) {
+
+		if len(instance.Status.StatefulSetStatuses) > len(instance.Spec.LaunchContainers) {
+			for key := len(instance.Spec.LaunchContainers); key < len(instance.Status.StatefulSetStatuses); key++ {
+				statefulSet := appsv1.StatefulSet{}
+				err := r.Get(ctx, *instance.GetStatefulSetMetadata(key), &statefulSet)
+				if err != nil {
+					return err
+				}
+
+				err = r.Delete(ctx, &statefulSet)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
 		for key := range instance.Spec.LaunchContainers {
 			ssStatus := robotv1alpha2.OwnedStatefulSetStatus{
 				Resource: robotv1alpha2.OwnedResourceStatus{
@@ -43,4 +64,6 @@ func (r *ROS2WorkloadReconciler) registerStatefulSets(instance *robotv1alpha2.RO
 		}
 		instance.Status.StatefulSetStatuses = ssStatuses
 	}
+
+	return nil
 }
diff --git a/pkg/controllers/v1alpha2/production/ros2_workload/ros2workload_controller.go b/pkg/controllers/v1alpha2/production/ros2_workload/ros2workload_controller.go
@@ -63,7 +63,10 @@ func (r *ROS2WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
-	r.reconcileRegisterResources(instance)
+	err = r.reconcileRegisterResources(ctx, instance)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
 
 	err = r.reconcileUpdateInstanceStatus(ctx, instance)
 	if err != nil {
@@ -93,10 +96,17 @@ func (r *ROS2WorkloadReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	return result, nil
 }
 
-func (r *ROS2WorkloadReconciler) reconcileRegisterResources(instance *robotv1alpha2.ROS2Workload) error {
+func (r *ROS2WorkloadReconciler) reconcileRegisterResources(ctx context.Context, instance *robotv1alpha2.ROS2Workload) error {
+
+	err := r.registerPVCs(ctx, instance)
+	if err != nil {
+		return err
+	}
 
-	r.registerPVCs(instance)
-	r.registerStatefulSets(instance)
+	err = r.registerStatefulSets(ctx, instance)
+	if err != nil {
+		return err
+	}
 
 	return nil
 }