Use findObjectsForSrc to watch the referenced topology

With the goal of deduplicating the current code as much as possible and help spreading the pattern across the operators, this patch adds the topologyRef dynamic field to the existing glanceAPIWatchList. This allows to both index the field (and use fieldSelector when the GlanceAPI CR is retrieved), and remove the topologyFn/tpFn functions. The watcher is now based on GenerationChangedPredicate, which only triggers a reconcile when .Spec for the referenced topology changes. Signed-off-by: Francesco Pantano <[email protected]>
openstack-k8s-operators · Jan 15, 2025 · c45e9cd · c45e9cd
1 parent 262fd74
commit c45e9cd
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 85 deletions.
diff --git a/api/go.mod b/api/go.mod
@@ -71,6 +71,4 @@ require (
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 //allow-merging
 
-replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/fmount/infra-operator/apis v0.0.0-20250109124018-4262fdefc70b //allow-merging
-
-replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/fmount/lib-common/modules/common v0.0.0-20241217100632-a2c8ea43c395 //allow-merging
+replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/fmount/infra-operator/apis v0.0.0-20250115113016-0c213529d253 //allow-merging
diff --git a/api/go.sum b/api/go.sum
@@ -12,8 +12,6 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
-github.com/fmount/lib-common/modules/common v0.0.0-20241217100632-a2c8ea43c395 h1:FTnFgkzbg5agJorJB4wXfU4LtW8xfGAsozo8It1i6vU=
-github.com/fmount/lib-common/modules/common v0.0.0-20241217100632-a2c8ea43c395/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
@@ -77,6 +75,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6Beb1gQ96Ptej9AE/BvwCBiRj1E=
 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=
+github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:YpNTuJhDWhbXM50O3qBkhO7M+OOyRmWkNVmJ4y3cyFs=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e h1:Qz0JFEoRDUyjEWorNY3LggwxTsmpMtQkcpmZDQulGHQ=
 github.com/openstack-k8s-operators/lib-common/modules/storage v0.5.1-0.20241216113837-d172b3ac0f4e/go.mod h1:tfgBeLRqmlH/NQkLPe7396rj+t0whv2wPuMb8Ttvh8w=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

diff --git a/controllers/glance_common.go b/controllers/glance_common.go
@@ -49,6 +49,7 @@ const (
 	caBundleSecretNameField = ".spec.tls.caBundleSecretName"
 	tlsAPIInternalField     = ".spec.tls.api.internal.secretName"
 	tlsAPIPublicField       = ".spec.tls.api.public.secretName"
+	topologyField           = ".spec.topologyRef.Name"
 )
 
 var (
@@ -60,6 +61,7 @@ var (
 		caBundleSecretNameField,
 		tlsAPIInternalField,
 		tlsAPIPublicField,
+		topologyField,
 	}
 )
 

diff --git a/controllers/glanceapi_controller.go b/controllers/glanceapi_controller.go
@@ -22,7 +22,6 @@ import (
 	"strings"
 
 	batchv1 "k8s.io/api/batch/v1"
-	"k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
@@ -34,7 +33,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -246,6 +244,18 @@ func (r *GlanceAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		return err
 	}
 
+	// index topologyField
+	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &glancev1.GlanceAPI{}, topologyField, func(rawObj client.Object) []string {
+		// Extract the topology name from the spec, if one is provided
+		cr := rawObj.(*glancev1.GlanceAPI)
+		if cr.Spec.Topology == nil {
+			return nil
+		}
+		return []string{cr.Spec.Topology.Name}
+	}); err != nil {
+		return err
+	}
+
 	// Watch for changes to any CustomServiceConfigSecrets. Global secrets
 	svcSecretFn := func(_ context.Context, o client.Object) []reconcile.Request {
 		var namespace string = o.GetNamespace()
@@ -336,44 +346,6 @@ func (r *GlanceAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		}
 		return nil
 	}
-	tpFn := predicate.Funcs{
-		UpdateFunc: func(e event.UpdateEvent) bool {
-			oldObj := e.ObjectOld.(*topologyv1.Topology)
-			newObj := e.ObjectNew.(*topologyv1.Topology)
-			// Compare spec
-			return !equality.Semantic.DeepEqual(oldObj.Spec, newObj.Spec)
-		},
-	}
-
-	topologyFn := func(_ context.Context, o client.Object) []reconcile.Request {
-		result := []reconcile.Request{}
-		// get all GlanceAPIs CRs
-		glanceAPIs := &glancev1.GlanceAPIList{}
-		listOpts := []client.ListOption{
-			client.InNamespace(o.GetNamespace()),
-		}
-		if err := r.Client.List(context.Background(), glanceAPIs, listOpts...); err != nil {
-			r.Log.Error(err, "Unable to retrieve GlanceAPI CRs %w")
-			return nil
-		}
-
-		for _, cr := range glanceAPIs.Items {
-			if cr.Spec.Topology != nil {
-				if o.GetName() == cr.Spec.Topology.Name {
-					name := client.ObjectKey{
-						Namespace: o.GetNamespace(),
-						Name:      cr.Name,
-					}
-					r.Log.Info(fmt.Sprintf("Topology %s is used by GlanceAPI CR %s", o.GetName(), cr.Name))
-					result = append(result, reconcile.Request{NamespacedName: name})
-				}
-			}
-		}
-		if len(result) > 0 {
-			return result
-		}
-		return nil
-	}
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&glancev1.GlanceAPI{}).
@@ -393,8 +365,8 @@ func (r *GlanceAPIReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Watches(&memcachedv1.Memcached{},
 			handler.EnqueueRequestsFromMapFunc(memcachedFn)).
 		Watches(&topologyv1.Topology{},
-			handler.EnqueueRequestsFromMapFunc(topologyFn),
-			builder.WithPredicates(tpFn)).
+			handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc),
+			builder.WithPredicates(predicate.GenerationChangedPredicate{})).
 		Complete(r)
 }
 
@@ -857,7 +829,8 @@ func (r *GlanceAPIReconciler) reconcileNormal(
 	// When the Topology CR reference is updated and the current GlanceAPI
 	// switches to a new Topology, remove the finalizer from the previous
 	// Topology
-	if instance.Status.LastAppliedTopology != "" {
+	if instance.Spec.Topology == nil ||
+		(instance.Spec.Topology.Name != instance.Status.LastAppliedTopology) {
 		_, err = r.ensureDeletedTopology(ctx, instance, helper)
 		if err != nil {
 			return ctrl.Result{}, err
@@ -1564,19 +1537,21 @@ func (r *GlanceAPIReconciler) ensureDeletedTopology(
 ) (ctrl.Result, error) {
 	ns := instance.Namespace
 
-	// no Topology is currently passed to the GlanceAPI, and it was not used
-	// before
-	if instance.Spec.Topology == nil && instance.Status.LastAppliedTopology == "" {
+	// no Topology is passed to the GlanceAPI, and it was not used before
+	if instance.Status.LastAppliedTopology == "" {
 		return ctrl.Result{}, nil
 	}
+
+	// Topology is referenced in the .Spec, check the namespace
 	if instance.Spec.Topology != nil {
-		// Check namespace and set name
+		// Check namespace
 		if instance.Spec.Topology.Namespace != "" {
 			ns = instance.Spec.Topology.Namespace
 		}
 	}
 
 	name := instance.Status.LastAppliedTopology
+
 	// Remove the finalizer from the Topology CR
 	topology, _, err := topologyv1.GetTopologyByName(
 		ctx,
@@ -1585,19 +1560,16 @@ func (r *GlanceAPIReconciler) ensureDeletedTopology(
 		ns,
 	)
 
-	if k8s_errors.IsNotFound(err) {
-		return ctrl.Result{}, nil
-	}
 	if err != nil && !k8s_errors.IsNotFound(err) {
 		return ctrl.Result{}, err
 	}
-	if err == nil {
+	if !k8s_errors.IsNotFound(err) {
 		if controllerutil.RemoveFinalizer(topology, fmt.Sprintf("%s-%s", h.GetFinalizer(), instance.APIName())) {
 			err = r.Update(ctx, topology)
 			if err != nil && !k8s_errors.IsNotFound(err) {
 				return ctrl.Result{}, err
 			}
-			util.LogForObject(h, "Removed finalizer from Topology", instance)
+			util.LogForObject(h, "Removed finalizer from Topology", topology)
 		}
 	}
 	return ctrl.Result{}, err

diff --git a/go.mod b/go.mod
@@ -90,6 +90,4 @@ replace github.com/openstack-k8s-operators/glance-operator/api => ./api
 // must consistent within modules and service operators
 replace github.com/openshift/api => github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 //allow-merging
 
-replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/fmount/infra-operator/apis v0.0.0-20250109124018-4262fdefc70b //allow-merging
-
-replace github.com/openstack-k8s-operators/lib-common/modules/common => github.com/fmount/lib-common/modules/common v0.0.0-20241217100632-a2c8ea43c395 //allow-merging
+replace github.com/openstack-k8s-operators/infra-operator/apis => github.com/fmount/infra-operator/apis v0.0.0-20250115113016-0c213529d253 //allow-merging
diff --git a/go.sum b/go.sum
@@ -12,6 +12,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ
 github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg=
 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ=
+github.com/fmount/infra-operator/apis v0.0.0-20250115113016-0c213529d253 h1:l2LQqMkbUYJC7KNm+spvhB45aedZ1msV9h75OmzhclI=
+github.com/fmount/infra-operator/apis v0.0.0-20250115113016-0c213529d253/go.mod h1:TDaE7BVQvJwJGFm33R6xcPTeF8LKAnMh+a1ho+YqJHs=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
@@ -80,8 +82,6 @@ github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094 h1:J1wuGhVxpsHykZBa6
 github.com/openshift/api v0.0.0-20240830023148-b7d0481c9094/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
 github.com/openstack-k8s-operators/cinder-operator/api v0.5.1-0.20241217072755-fb4d39411ad2 h1:iY0RYw5hyeaou7ujKCrkYpgebIyh5O5FRsvkMwSZdNk=
 github.com/openstack-k8s-operators/cinder-operator/api v0.5.1-0.20241217072755-fb4d39411ad2/go.mod h1:GoZVwi0mFiqivii2K+EBYW0meEv/X8txqqpBRjxDVbc=
-github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20250108092548-58707fa645ce h1:2F25J61AF3STCfW69SSQ+ribEq0fcTShZjCbb4CA6AU=
-github.com/openstack-k8s-operators/infra-operator/apis v0.5.1-0.20250108092548-58707fa645ce/go.mod h1:TDaE7BVQvJwJGFm33R6xcPTeF8LKAnMh+a1ho+YqJHs=
 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20250107165241-16c3ed8e549f h1:jlUo93FAwlDll1bJRxJO5B1Vi3t3wCoHQuy5HEO96ME=
 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20250107165241-16c3ed8e549f/go.mod h1:CyuEOM1TpXKNUR1n8cudNtRzTEwkzv90JFkpDPPId8E=
 github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241216113837-d172b3ac0f4e h1:hf4kVQBkyG79WcHBxdQ25QrDBbGFdarebS1Tc0Xclq4=

diff --git a/pkg/glanceapi/statefulset.go b/pkg/glanceapi/statefulset.go
@@ -50,7 +50,7 @@ func StatefulSet(
 	labels map[string]string,
 	annotations map[string]string,
 	privileged bool,
-	topologyOverride *topologyv1.Topology,
+	topology *topologyv1.Topology,
 ) (*appsv1.StatefulSet, error) {
 	userID := glance.GlanceUID
 	startupProbe := &corev1.Probe{
@@ -76,13 +76,6 @@ func StatefulSet(
 	glanceURIScheme := corev1.URISchemeHTTP
 	tlsEnabled := instance.Spec.TLS.API.Enabled(service.EndpointPublic)
 
-	// initialize an Affinity Object
-	aff := &affinity.OverrideSpec{
-		PodAffinity:     nil,
-		PodAntiAffinity: nil,
-		NodeAffinity:    nil,
-	}
-
 	// initialize a Topology Object
 	tplg := []corev1.TopologySpreadConstraint{}
 
@@ -307,26 +300,28 @@ func StatefulSet(
 		statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector
 	}
 
-	if topologyOverride != nil {
-		ts := topologyOverride.Spec
+	if topology != nil {
+		ts := topology.Spec
 		if ts.TopologySpreadConstraint == nil {
 			ts.TopologySpreadConstraint = &tplg
 		} else {
-			statefulset.Spec.Template.Spec.TopologySpreadConstraints = *topologyOverride.Spec.TopologySpreadConstraint
+			statefulset.Spec.Template.Spec.TopologySpreadConstraints = *topology.Spec.TopologySpreadConstraint
+		}
+		if ts.Affinity != nil {
+			statefulset.Spec.Template.Spec.Affinity = ts.Affinity
 		}
-		aff = topologyOverride.Spec.APIAffinity
+	} else {
+		// If possible two pods of the same service should not
+		// run on the same worker node. If this is not possible
+		// the get still created on the same worker node.
+		statefulset.Spec.Template.Spec.Affinity = affinity.DistributePods(
+			common.AppSelector,
+			[]string{
+				glance.ServiceName,
+			},
+			corev1.LabelHostname,
+		)
 	}
 
-	// If possible two pods of the same service should not
-	// run on the same worker node. If this is not possible
-	// the get still created on the same worker node.
-	statefulset.Spec.Template.Spec.Affinity, err = affinity.DistributePods(
-		common.AppSelector,
-		[]string{
-			glance.ServiceName,
-		},
-		corev1.LabelHostname,
-		aff,
-	)
 	return statefulset, err
 }