From 1dbc770195c9cdd994ce7eb2a8247267a766ca57 Mon Sep 17 00:00:00 2001 From: Christina Xu Date: Mon, 13 Jan 2025 15:51:53 -0500 Subject: [PATCH] Add logic to add/update status --- .../v1alpha1/guardrailsorchestrator_types.go | 54 ++++++-- api/gorch/v1alpha1/zz_generated.deepcopy.go | 19 ++- ...pendatahub.io_guardrailsorchestrators.yaml | 41 +++--- .../guardrailsorchestrator_controller.go | 41 +++--- .../guardrailsorchestrator_controller_test.go | 2 +- controllers/gorch/status.go | 121 ++++++++++++++++++ 6 files changed, 215 insertions(+), 63 deletions(-) create mode 100644 controllers/gorch/status.go diff --git a/api/gorch/v1alpha1/guardrailsorchestrator_types.go b/api/gorch/v1alpha1/guardrailsorchestrator_types.go index 84db74c..eeb5c4e 100644 --- a/api/gorch/v1alpha1/guardrailsorchestrator_types.go +++ b/api/gorch/v1alpha1/guardrailsorchestrator_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha1 import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -70,23 +72,49 @@ type GuardrailsOrchestratorSpec struct { TLS string `json:"tls,omitempty"` } -// const ( -// OrchestratorConditionReconciled ConditionType = "Reconciled" -// // OrchestratorConditionReady represents the fact that the orchestrator's components are ready -// OrchestratorConditionReady ConditionType = "Ready" -// OrchestratorDeploymentNotReady ConditionReason = "DeploymentNotReady" -// OrchestratorInferenceServiceNotReady ConditionReason = "InferenceServiceNotReady" -// OrchestratorRouteNotAdmitted ConditionReason = "RouteNotAdmitted" -// OrchestratorHealthy ConditionReason = "Healthy" -// OrchestratorReadinessCheckFailed ConditionReason = "ReadinessCheckFailed" -// ) - type GuardrailsOrchestratorStatus struct { - Conditions []metav1.Condition `json:"conditions"` + Conditions []GuardrailsOrchestratorCondition `json:"conditions,omitempty"` +} + +var testTime *time.Time + +func (in *GuardrailsOrchestratorStatus) SetConditions(condition GuardrailsOrchestratorCondition) { + var now time.Time + if testTime == nil { + now = time.Now() + } else { + now = *testTime + } + + lastTransitionTime := metav1.NewTime(now.Truncate(time.Second)) + + for i, prevCondition := range in.Conditions { + if prevCondition.Type == condition.Type { + if prevCondition.Status != condition.Status { + condition.LastTransitionTime = lastTransitionTime + } else { + condition.LastTransitionTime = prevCondition.LastTransitionTime + } + in.Conditions[i] = condition + return + } + } + condition.LastTransitionTime = lastTransitionTime + in.Conditions = append(in.Conditions, condition) +} + +type GuardrailsOrchestratorCondition struct { + Type string `json:"type"` + Status metav1.ConditionStatus `json:"status"` + // +optional + Reason string `json:"reason,omitmempty"` + // +optional + Message string `json:"message,omitempty"` + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` } // +kubebuilder:object:root=true -// +kubebuilder:subresource:status // GuardrailsOrchestrator is the Schema for the guardrailsorchestrators API. type GuardrailsOrchestrator struct { diff --git a/api/gorch/v1alpha1/zz_generated.deepcopy.go b/api/gorch/v1alpha1/zz_generated.deepcopy.go index 5518a87..6c1756e 100644 --- a/api/gorch/v1alpha1/zz_generated.deepcopy.go +++ b/api/gorch/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,6 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -100,6 +99,22 @@ func (in *GuardrailsOrchestrator) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GuardrailsOrchestratorCondition) DeepCopyInto(out *GuardrailsOrchestratorCondition) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GuardrailsOrchestratorCondition. +func (in *GuardrailsOrchestratorCondition) DeepCopy() *GuardrailsOrchestratorCondition { + if in == nil { + return nil + } + out := new(GuardrailsOrchestratorCondition) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GuardrailsOrchestratorList) DeepCopyInto(out *GuardrailsOrchestratorList) { *out = *in @@ -163,7 +178,7 @@ func (in *GuardrailsOrchestratorStatus) DeepCopyInto(out *GuardrailsOrchestrator *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]GuardrailsOrchestratorCondition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/config/crd/bases/trustyai.opendatahub.io_guardrailsorchestrators.yaml b/config/crd/bases/trustyai.opendatahub.io_guardrailsorchestrators.yaml index b2d112f..64e489c 100644 --- a/config/crd/bases/trustyai.opendatahub.io_guardrailsorchestrators.yaml +++ b/config/crd/bases/trustyai.opendatahub.io_guardrailsorchestrators.yaml @@ -52,8 +52,6 @@ spec: type: string port: type: integer - protocol: - type: string tls: properties: ca_cert_path: @@ -73,7 +71,6 @@ spec: required: - hostname - port - - protocol type: object required: - provider @@ -94,8 +91,6 @@ spec: type: string port: type: integer - protocol: - type: string tls: properties: ca_cert_path: @@ -115,7 +110,6 @@ spec: required: - hostname - port - - protocol type: object type: type: string @@ -137,8 +131,6 @@ spec: type: string port: type: integer - protocol: - type: string tls: properties: ca_cert_path: @@ -158,7 +150,6 @@ spec: required: - hostname - port - - protocol type: object required: - provider @@ -177,21 +168,27 @@ spec: - replicas type: object status: - description: GuardrailsOrchestratorStatus defines the observed state of - GuardrailsOrchestrator. properties: - condition: - description: |- - INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - Important: Run "make" to regenerate code after modifying this file - type: string - ready: - type: boolean - required: - - ready + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + required: + - status + - type + type: object + type: array type: object type: object served: true storage: true - subresources: - status: {} diff --git a/controllers/gorch/guardrailsorchestrator_controller.go b/controllers/gorch/guardrailsorchestrator_controller.go index e88a701..5161258 100644 --- a/controllers/gorch/guardrailsorchestrator_controller.go +++ b/controllers/gorch/guardrailsorchestrator_controller.go @@ -23,7 +23,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -38,11 +37,6 @@ import ( gorchv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/gorch/v1alpha1" ) -const ( - typeDegradedOrchestrator = "Degraded" - typeAvailableOrchestrator = "Available" -) - // GuardrailsOrchestratorReconciler reconciles a GuardrailsOrchestrator object type GuardrailsOrchestratorReconciler struct { client.Client @@ -110,9 +104,9 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct if controllerutil.ContainsFinalizer(orchestrator, finalizerName) { log.Info("Performing Finalizer Operations for GuardrailsOrchestrator before delete CR") - meta.SetStatusCondition(&orchestrator.Status.Conditions, metav1.Condition{Type: typeDegradedOrchestrator, - Status: metav1.ConditionUnknown, Reason: "Finalizing", - Message: fmt.Sprintf("Performing finalizer operations for the custom resource: %s", orchestrator.Name)}) + condition := gorchv1alpha1.GuardrailsOrchestratorCondition{Type: typeDegradedOrchestrator, Status: metav1.ConditionUnknown, Reason: "Finalizing", + Message: fmt.Sprintf("Performing finalizer operations for the custom resource: %s", orchestrator.Name)} + orchestrator.Status.SetConditions(condition) if err := r.Status().Update(ctx, orchestrator); err != nil { log.Error(err, "Failed to update GuardrailsOrchestrator status") @@ -128,10 +122,11 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct log.Error(err, "Failed to re-fetch GuardrailsOrchestrator") return ctrl.Result{}, err } + condition.Status = metav1.ConditionTrue + condition.Reason = "Finalizing" + condition.Message = fmt.Sprintf("Finalizer operations for custom resource %s were sucessfully accomplished", orchestrator.Name) - meta.SetStatusCondition(&orchestrator.Status.Conditions, metav1.Condition{Type: typeDegradedOrchestrator, - Status: metav1.ConditionTrue, Reason: "Finalizing", - Message: fmt.Sprintf("Finalizer operations for custom resource %s were sucessfully accomplished", orchestratorName)}) + orchestrator.Status.SetConditions(condition) if err := r.Status().Update(ctx, orchestrator); err != nil { log.Error(err, "Failed to update GuardrailsOrchestrator status") @@ -170,16 +165,17 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct existingDeployment := &appsv1.Deployment{} err = r.Get(ctx, types.NamespacedName{Name: orchestratorName, Namespace: orchestrator.Namespace}, existingDeployment) if err != nil && errors.IsNotFound(err) { + // Create a new deployment deployment := r.createDeployment(ctx, orchestrator) log.Info("Creating a new Deployment", "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name) err = r.Create(ctx, deployment) if err != nil { log.Error(err, "Failed to create new Deployment", "Deployment.Namespace", deployment.Namespace, "Deployment.Name", deployment.Name) - return ctrl.Result{}, err + return ctrl.Result{}, r.statusUpdateError(ctx, orchestrator, r.setStatusFailed("FailedToCreateDeployment"), fmt.Errorf("failed to get %v Deployment resource, err %w", orchestratorName, err)) } } else if err != nil { log.Error(err, "Failed to get Deployment") - return ctrl.Result{}, err + return ctrl.Result{}, r.statusUpdateError(ctx, orchestrator, r.setStatusFailed("FailedToGetDeployment"), fmt.Errorf("failed to get %v Deployment resource, err %w", orchestratorName, err)) } existingService := &corev1.Service{} @@ -191,11 +187,11 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct err = r.Create(ctx, service) if err != nil { log.Error(err, "Failed to create new Service", "Service.Namespace", service.Namespace, "Service.Name", service.Name) - return ctrl.Result{}, err + return ctrl.Result{}, r.statusUpdateError(ctx, orchestrator, r.setStatusFailed("FailedToCreateService"), fmt.Errorf("failed to get %v Service resource, err %w", orchestratorName, err)) } } else if err != nil { log.Error(err, "Failed to get Service") - return ctrl.Result{}, err + return ctrl.Result{}, r.statusUpdateError(ctx, orchestrator, r.setStatusFailed("FailedToGetService"), fmt.Errorf("failed to get %v Service resource, err %w", orchestratorName, err)) } existingRoute := &routev1.Route{} @@ -207,21 +203,16 @@ func (r *GuardrailsOrchestratorReconciler) Reconcile(ctx context.Context, req ct err = r.Create(ctx, route) if err != nil { log.Error(err, "Failed to create new Route", "Route.Namespace", route.Namespace, "Route.Name", route.Name) - return ctrl.Result{}, err + return ctrl.Result{}, r.statusUpdateError(ctx, orchestrator, r.setStatusFailed("FailedToCreateRoute"), fmt.Errorf("failed to get %v Route resource, err %w", orchestratorName, err)) } } else if err != nil { log.Error(err, "Failed to get Route") - return ctrl.Result{}, err + return ctrl.Result{}, r.statusUpdateError(ctx, orchestrator, r.setStatusFailed("FailedToGetRoute"), fmt.Errorf("failed to get %v Route resource, err %w", orchestratorName, err)) } - meta.SetStatusCondition(&orchestrator.Status.Conditions, metav1.Condition{Type: typeAvailableOrchestrator, Status: metav1.ConditionTrue, - Reason: "Available", Message: "GuardrailsOrchestrator is available"}) - if err := r.Status().Update(ctx, orchestrator); err != nil { - log.Error(err, "Failed to update GuardrailsOrchestrator status") - return ctrl.Result{}, err - } + updateStatusConditions(orchestrator, r.Client, statusReady()) - return ctrl.Result{}, nil + return ctrl.Result{}, r.Get(ctx, req.NamespacedName, orchestrator) } func (r *GuardrailsOrchestratorReconciler) deleteRoute(ctx context.Context, orchestrator gorchv1alpha1.GuardrailsOrchestrator) (err error) { diff --git a/controllers/gorch/guardrailsorchestrator_controller_test.go b/controllers/gorch/guardrailsorchestrator_controller_test.go index a648c83..9350d93 100644 --- a/controllers/gorch/guardrailsorchestrator_controller_test.go +++ b/controllers/gorch/guardrailsorchestrator_controller_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" routev1 "github.com/openshift/api/route/v1" + "sigs.k8s.io/controller-runtime/pkg/reconcile" // "golang.org/x/sys/unix" appsv1 "k8s.io/api/apps/v1" @@ -30,7 +31,6 @@ import ( "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/reconcile" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/controllers/gorch/status.go b/controllers/gorch/status.go new file mode 100644 index 0000000..b1d5528 --- /dev/null +++ b/controllers/gorch/status.go @@ -0,0 +1,121 @@ +package gorch + +import ( + "context" + "time" + + gorchv1alpha1 "github.com/trustyai-explainability/trustyai-service-operator/api/gorch/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + typeDegradedOrchestrator = "Degraded" + typeAvailableOrchestrator = "Available" +) + +type updateStatusFunc func(orchestrator *gorchv1alpha1.GuardrailsOrchestrator, message string) error + +func setStatusConditions(conditions []gorchv1alpha1.GuardrailsOrchestratorCondition, newConditions ...gorchv1alpha1.GuardrailsOrchestratorCondition) ([]gorchv1alpha1.GuardrailsOrchestratorCondition, bool) { + var atLeastOneUpdated bool + var updated bool + for _, cond := range newConditions { + conditions, updated = updateStatusCondition(conditions, cond) + atLeastOneUpdated = atLeastOneUpdated || updated + } + + return conditions, atLeastOneUpdated +} + +var testTime *time.Time + +// updateStatusConditions updates the status of the orchestrator with the new conditions +func updateStatusConditions(orchestrator *gorchv1alpha1.GuardrailsOrchestrator, client client.Client, newConditions ...gorchv1alpha1.GuardrailsOrchestratorCondition) error { + var updated bool + log := log.FromContext(context.Background()) + orchestrator.Status.Conditions, updated = setStatusConditions(orchestrator.Status.Conditions, newConditions...) + if !updated { + log.Info("GuardrailsOrchestrator status conditions not changed") + return nil + } + return client.Status().Update(context.TODO(), orchestrator) +} + +// updateStatusCondition updates the status of the orchestrator with the new condition. +func updateStatusCondition(conditions []gorchv1alpha1.GuardrailsOrchestratorCondition, newCondition gorchv1alpha1.GuardrailsOrchestratorCondition) ([]gorchv1alpha1.GuardrailsOrchestratorCondition, bool) { + var now time.Time + if testTime == nil { + now = time.Now() + } else { + now = *testTime + } + + newCondition.LastTransitionTime = metav1.NewTime(now.Truncate(time.Second)) + + // If the condition doesn't exist, add it + if conditions == nil { + return []gorchv1alpha1.GuardrailsOrchestratorCondition{newCondition}, true + } + for i, cond := range conditions { + if cond.Type == newCondition.Type { + // Case 1: The condition has not changed + if cond.Status == newCondition.Status && + cond.Reason == newCondition.Reason && + cond.Message == newCondition.Message { + return conditions, false + } + + // Case 2: The condition status has changed + if newCondition.Status == cond.Status { + newCondition.LastTransitionTime = cond.LastTransitionTime + } + // Case 3: The condition has changed, create a new slice with the updated condition + res := make([]gorchv1alpha1.GuardrailsOrchestratorCondition, len(conditions)) + copy(res, conditions) + res[i] = newCondition + return res, true + } + } + return append(conditions, newCondition), true +} + +// statusUpdateError updates the status of the orchestrator with the error message if the error is not nil. +func (r *GuardrailsOrchestratorReconciler) statusUpdateError(ctx context.Context, orchestrator *gorchv1alpha1.GuardrailsOrchestrator, updateStatus updateStatusFunc, err error) error { + log := log.FromContext(ctx) + if err == nil { + return nil + } + if err := updateStatus(orchestrator, err.Error()); err != nil { + log.Error(err, "Failed to update GuardrailsOrchestrator status") + } + return err +} + +// setStatusFailed handles the case where one of the orchestrator's components is not available. +func (r *GuardrailsOrchestratorReconciler) setStatusFailed(reason string) updateStatusFunc { + return func(orchestrator *gorchv1alpha1.GuardrailsOrchestrator, message string) error { + return updateStatusConditions( + orchestrator, + r.Client, + statusNotReady(reason, message), + ) + } +} + +func statusReady() gorchv1alpha1.GuardrailsOrchestratorCondition { + return gorchv1alpha1.GuardrailsOrchestratorCondition{ + Type: typeAvailableOrchestrator, + Status: metav1.ConditionTrue, + Reason: "OrchestratorAvailable", + } +} + +func statusNotReady(reason, message string) gorchv1alpha1.GuardrailsOrchestratorCondition { + return gorchv1alpha1.GuardrailsOrchestratorCondition{ + Type: typeAvailableOrchestrator, + Status: metav1.ConditionFalse, + Reason: reason, + Message: message, + } +}