From 48a6210f967d276e32ea8b1bfa9b379803800242 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Tue, 10 Dec 2024 17:10:36 +0100 Subject: [PATCH] Add initial watcher api conf generation Implement an initial version of watcher config generation. This change adds a watcher config template and generates a secret with the templated config. Some fields that require changes in the watcher controller like the transporturl and memcached servers. This change also moidifies the WatcherAPI functional tests so the WatcherAPI instances use a different name that the Watcher one, so it's easier to debug. --- .../watcher.openstack.org_watcherapis.yaml | 11 +- api/bases/watcher.openstack.org_watchers.yaml | 11 +- api/v1beta1/common_types.go | 3 +- api/v1beta1/watcher_types.go | 5 + api/v1beta1/watcherapi_types.go | 5 + .../watcher.openstack.org_watcherapis.yaml | 11 +- .../bases/watcher.openstack.org_watchers.yaml | 11 +- config/rbac/role.yaml | 25 +++ .../samples/watcher_v1beta1_watcherapi.yaml | 1 + controllers/watcher_common.go | 79 ++++++++ controllers/watcherapi_controller.go | 93 ++++++++-- go.mod | 2 +- main.go | 2 + templates/watcher/config/00-default.conf | 43 +++++ templates/watcherapi | 1 + tests/functional/base_test.go | 5 +- tests/functional/sample_test.go | 2 +- tests/functional/suite_test.go | 7 + tests/functional/watcher_test_data.go | 10 + .../functional/watcherapi_controller_test.go | 173 +++++++++++++++--- .../default/watcher-api/03-assert.yaml | 10 + .../watcher-api/03-deploy-watcher-api.yaml | 2 + 22 files changed, 462 insertions(+), 50 deletions(-) create mode 100644 templates/watcher/config/00-default.conf create mode 120000 templates/watcherapi diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index a7e0f63..9068f52 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -49,6 +49,10 @@ spec: MariaDB instance name Required to use the mariadb-operator instance to create the DB and user type: string + memcachedInstance: + description: MemcachedInstance is the name of the Memcached CR that + all watcher service will use. + type: string passwordSelectors: default: service: WatcherPassword @@ -66,11 +70,14 @@ spec: type: string serviceUser: default: watcher - description: ServiceUser - optional username used for this service - to register in keystone + description: |- + ServiceUser - optional username used for this service to register in + keystone type: string required: - databaseInstance + - memcachedInstance + - passwordSelectors - secret type: object status: diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index 28bae46..ac57c5b 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -49,6 +49,11 @@ spec: MariaDB instance name Required to use the mariadb-operator instance to create the DB and user type: string + memcachedInstance: + default: memcached + description: MemcachedInstance is the name of the Memcached CR that + all watcher service will use. + type: string passwordSelectors: default: service: WatcherPassword @@ -73,11 +78,13 @@ spec: type: string serviceUser: default: watcher - description: ServiceUser - optional username used for this service - to register in keystone + description: |- + ServiceUser - optional username used for this service to register in + keystone type: string required: - databaseInstance + - passwordSelectors - rabbitMqClusterName type: object status: diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 611cbc6..7f685ce 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -21,7 +21,8 @@ type WatcherCommon struct { // +kubebuilder:validation:Optional // +kubebuilder:default=watcher - // ServiceUser - optional username used for this service to register in keystone + // ServiceUser - optional username used for this service to register in + // keystone ServiceUser string `json:"serviceUser"` // +kubebuilder:validation:Optional diff --git a/api/v1beta1/watcher_types.go b/api/v1beta1/watcher_types.go index 1dafb90..e7aa51b 100644 --- a/api/v1beta1/watcher_types.go +++ b/api/v1beta1/watcher_types.go @@ -27,6 +27,11 @@ type WatcherSpec struct { // Important: Run "make" to regenerate code after modifying this file WatcherTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=memcached + // MemcachedInstance is the name of the Memcached CR that all watcher service will use. + MemcachedInstance string `json:"memcachedInstance"` } // WatcherStatus defines the observed state of Watcher diff --git a/api/v1beta1/watcherapi_types.go b/api/v1beta1/watcherapi_types.go index 90cacf2..33a3f95 100644 --- a/api/v1beta1/watcherapi_types.go +++ b/api/v1beta1/watcherapi_types.go @@ -27,9 +27,14 @@ type WatcherAPISpec struct { // Important: Run "make" to regenerate code after modifying this file WatcherCommon `json:",inline"` + // +kubebuilder:validation:Required // Secret containing all passwords / keys needed Secret string `json:"secret"` + + // +kubebuilder:validation:Required + // MemcachedInstance is the name of the Memcached CR that all watcher service will use. + MemcachedInstance string `json:"memcachedInstance"` } // WatcherAPIStatus defines the observed state of WatcherAPI diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index a7e0f63..9068f52 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -49,6 +49,10 @@ spec: MariaDB instance name Required to use the mariadb-operator instance to create the DB and user type: string + memcachedInstance: + description: MemcachedInstance is the name of the Memcached CR that + all watcher service will use. + type: string passwordSelectors: default: service: WatcherPassword @@ -66,11 +70,14 @@ spec: type: string serviceUser: default: watcher - description: ServiceUser - optional username used for this service - to register in keystone + description: |- + ServiceUser - optional username used for this service to register in + keystone type: string required: - databaseInstance + - memcachedInstance + - passwordSelectors - secret type: object status: diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index 28bae46..ac57c5b 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -49,6 +49,11 @@ spec: MariaDB instance name Required to use the mariadb-operator instance to create the DB and user type: string + memcachedInstance: + default: memcached + description: MemcachedInstance is the name of the Memcached CR that + all watcher service will use. + type: string passwordSelectors: default: service: WatcherPassword @@ -73,11 +78,13 @@ spec: type: string serviceUser: default: watcher - description: ServiceUser - optional username used for this service - to register in keystone + description: |- + ServiceUser - optional username used for this service to register in + keystone type: string required: - databaseInstance + - passwordSelectors - rabbitMqClusterName type: object status: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 3a073f5..5272bb1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -28,6 +28,14 @@ rules: - patch - update - watch +- apiGroups: + - keystone.openstack.org + resources: + - keystoneapis + verbs: + - get + - list + - watch - apiGroups: - keystone.openstack.org resources: @@ -82,6 +90,23 @@ rules: - patch - update - watch +- apiGroups: + - memcached.openstack.org + resources: + - memcacheds + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - memcached.openstack.org + resources: + - memcacheds/finalizers + verbs: + - patch + - update - apiGroups: - rabbitmq.openstack.org resources: diff --git a/config/samples/watcher_v1beta1_watcherapi.yaml b/config/samples/watcher_v1beta1_watcherapi.yaml index b593cf1..6184181 100644 --- a/config/samples/watcher_v1beta1_watcherapi.yaml +++ b/config/samples/watcher_v1beta1_watcherapi.yaml @@ -8,3 +8,4 @@ metadata: spec: databaseInstance: "openstack" secret: "osp-secret" + memcachedInstance: "memcached" diff --git a/controllers/watcher_common.go b/controllers/watcher_common.go index 6331edc..c2c38e3 100644 --- a/controllers/watcher_common.go +++ b/controllers/watcher_common.go @@ -15,7 +15,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/util" ) @@ -188,3 +192,78 @@ func ensureSecret( return hash, ctrl.Result{}, *secret, nil } + +func GenerateConfigsGeneric( + ctx context.Context, helper *helper.Helper, + instance client.Object, + envVars *map[string]env.Setter, + templateParameters map[string]interface{}, + customData map[string]string, + cmLabels map[string]string, + scripts bool, +) error { + + cms := []util.Template{ + // Templates where the watcher config is stored + { + Name: fmt.Sprintf("%s-config-data", instance.GetName()), + Namespace: instance.GetNamespace(), + Type: util.TemplateTypeConfig, + InstanceType: instance.GetObjectKind().GroupVersionKind().Kind, + ConfigOptions: templateParameters, + CustomData: customData, + Labels: cmLabels, + }, + } + if scripts { + cms = append(cms, util.Template{ + Name: fmt.Sprintf("%s-scripts", instance.GetName()), + Namespace: instance.GetNamespace(), + Type: util.TemplateTypeScripts, + InstanceType: instance.GetObjectKind().GroupVersionKind().Kind, + ConfigOptions: templateParameters, + Labels: cmLabels, + }) + } + return secret.EnsureSecrets(ctx, helper, instance, cms, envVars) +} + +// ensureMemcached - gets the Memcached instance cell specific used for nova services cache backend +func ensureMemcached( + ctx context.Context, + helper *helper.Helper, + namespaceName string, + memcachedName string, + conditionUpdater conditionUpdater, +) (*memcachedv1.Memcached, error) { + memcached, err := memcachedv1.GetMemcachedByName(ctx, helper, memcachedName, namespaceName) + if err != nil { + if k8s_errors.IsNotFound(err) { + conditionUpdater.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return nil, fmt.Errorf("memcached %s not found", memcachedName) + } + conditionUpdater.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return nil, err + } + + if !memcached.IsReady() { + conditionUpdater.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return nil, fmt.Errorf("memcached %s is not ready", memcachedName) + } + conditionUpdater.MarkTrue(condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + + return memcached, err +} diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index 19fee10..0486eac 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -30,9 +30,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/go-logr/logr" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" @@ -59,6 +64,11 @@ func (r *WatcherAPIReconciler) GetLogger(ctx context.Context) logr.Logger { //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis/status,verbs=get;update;patch //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis/finalizers,verbs=update //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; +//+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds/finalizers,verbs=update;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -138,6 +148,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, []string{ instance.Spec.PasswordSelectors.Service, + TransportURLSelector, }, helper.GetClient(), &instance.Status.Conditions, @@ -163,7 +174,26 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) // all our input checks out so report InputReady instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) - err = r.generateServiceConfigs(ctx, instance, secret, db, helper, &configVars) + memcached, err := ensureMemcached(ctx, helper, instance.Namespace, instance.Spec.MemcachedInstance, &instance.Status.Conditions) + + if err != nil { + return ctrl.Result{}, err + } + // Add finalizer to Memcached to prevent it from being deleted now that we're using it + if controllerutil.AddFinalizer(memcached, helper.GetFinalizer()) { + err := helper.GetClient().Update(ctx, memcached) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + + err = r.generateServiceConfigs(ctx, instance, secret, db, memcached, helper, &configVars) if err != nil { return ctrl.Result{}, err } @@ -181,25 +211,65 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // generateServiceConfigs - create Secret which holds the service configuration -// NOTE - jgilaber this function is WIP, currently implements a fraction of its -// functionality and will be expanded of further iteration to actually generate -// the service configs func (r *WatcherAPIReconciler) generateServiceConfigs( ctx context.Context, instance *watcherv1beta1.WatcherAPI, secret corev1.Secret, db *mariadbv1.Database, + memcachedInstance *memcachedv1.Memcached, helper *helper.Helper, envVars *map[string]env.Setter, ) error { Log := r.GetLogger(ctx) Log.Info("generateServiceConfigs - reconciling") - // replace by actual usage in future iterations - _ = db - _ = helper - _ = instance - _ = secret - _ = envVars + labels := labels.GetLabels(instance, labels.GetGroupLabel(watcher.ServiceName), map[string]string{}) + // jgilaber this might be wrong? we should probably get keystonapi in the + // watcher controller and set the url in the spec eventually? + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, helper, instance.Namespace, map[string]string{}) + // KeystoneAPI not available we should not aggregate the error and continue + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + "keystoneAPI not found")) + return err + } + keystoneInternalURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return err + } + // customData hold any customization for the service. + // NOTE jgilaber making an empty map for now, we'll probably want to + // implement CustomServiceConfig later + customData := map[string]string{} + + databaseAccount := db.GetAccount() + databaseSecret := db.GetSecret() + templateParameters := map[string]interface{}{ + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?charset=utf8&plugin=dbcounter", + databaseAccount.Spec.UserName, + string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]), + db.GetDatabaseHostname(), + watcher.DatabaseName, + ), + "KeystoneAuthURL": keystoneInternalURL, + "ServicePassword": string(secret.Data[instance.Spec.PasswordSelectors.Service]), + "ServiceUser": instance.Spec.ServiceUser, + "TransportURL": string(secret.Data[TransportURLSelector]), + "MemcachedServers": memcachedInstance.GetMemcachedServerListString(), + } - return nil + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("%s-%s.%s.svc", watcher.ServiceName, endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it below when implemented + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + + return GenerateConfigsGeneric(ctx, helper, instance, envVars, templateParameters, customData, labels, false) } func (r *WatcherAPIReconciler) reconcileDelete(ctx context.Context, instance *watcherv1beta1.WatcherAPI, helper *helper.Helper) (ctrl.Result, error) { @@ -221,6 +291,7 @@ func (r *WatcherAPIReconciler) initStatus(instance *watcherv1beta1.WatcherAPI) e condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyMessage), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), ) instance.Status.Conditions.Init(&cl) diff --git a/go.mod b/go.mod index 4d93654..a350da6 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( k8s.io/api v0.29.10 k8s.io/apimachinery v0.29.10 k8s.io/client-go v0.29.10 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.17.6 ) @@ -75,7 +76,6 @@ require ( k8s.io/component-base v0.29.10 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/main.go b/main.go index 084ea8c..ad482cf 100644 --- a/main.go +++ b/main.go @@ -37,6 +37,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" @@ -58,6 +59,7 @@ func init() { utilruntime.Must(mariadbv1.AddToScheme(scheme)) utilruntime.Must(rabbitmqv1.AddToScheme(scheme)) utilruntime.Must(keystonev1.AddToScheme(scheme)) + utilruntime.Must(memcachedv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } diff --git a/templates/watcher/config/00-default.conf b/templates/watcher/config/00-default.conf new file mode 100644 index 0000000..3d2d7c9 --- /dev/null +++ b/templates/watcher/config/00-default.conf @@ -0,0 +1,43 @@ +[DEFAULT] +state_path = /opt/stack/data/watcher +transport_url = {{ .TransportURL }} +control_exchange = watcher +debug = True + +[database] +connection = {{ .DatabaseConnection }} + +[oslo_policy] +policy_file = /etc/watcher/policy.yaml.sample + +[oslo_messaging_notifications] +driver = messagingv2 + +[keystone_authtoken] +memcached_servers = {{ .MemcachedServers }} +# TODO jgilaber implement handling this option when we add tls support +cafile = /opt/stack/data/ca-bundle.pem +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_url = {{ .KeystoneAuthURL }} +interface = internal +auth_type = password + +[watcher_clients_auth] +memcached_servers = {{ .MemcachedServers }} +# TODO jgilaber implement handling this option when we add tls support +# cafile = /opt/stack/data/ca-bundle.pem +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_url = {{ .KeystoneAuthURL }} +interface = internal +auth_type = password + +[oslo_concurrency] +lock_path = /var/lib/watcher/tmp diff --git a/templates/watcherapi b/templates/watcherapi new file mode 120000 index 0000000..cba20eb --- /dev/null +++ b/templates/watcherapi @@ -0,0 +1 @@ +watcher/ \ No newline at end of file diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index 69f4a92..5f09642 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -38,8 +38,9 @@ func GetDefaultWatcherSpec() map[string]interface{} { func GetDefaultWatcherAPISpec() map[string]interface{} { return map[string]interface{}{ - "databaseInstance": "openstack", - "secret": SecretName, + "databaseInstance": "openstack", + "secret": SecretName, + "memcachedInstance": "memcached", } } diff --git a/tests/functional/sample_test.go b/tests/functional/sample_test.go index 4efea79..91d6c57 100644 --- a/tests/functional/sample_test.go +++ b/tests/functional/sample_test.go @@ -72,7 +72,7 @@ var _ = Describe("Samples", func() { When("watcher_v1beta1_watcherapi.yaml sample is applied", func() { It("WatcherAPI is created", func() { - name := CreateWatcherAPIFromSample("watcher_v1beta1_watcherapi.yaml", watcherTest.Instance) + name := CreateWatcherAPIFromSample("watcher_v1beta1_watcherapi.yaml", watcherTest.WatcherAPI) GetWatcherAPI(name) }) }) diff --git a/tests/functional/suite_test.go b/tests/functional/suite_test.go index 84a41d2..63d0e88 100644 --- a/tests/functional/suite_test.go +++ b/tests/functional/suite_test.go @@ -28,6 +28,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" test "github.com/openstack-k8s-operators/lib-common/modules/test" @@ -67,6 +68,8 @@ const ( SecretName = "test-osp-secret" interval = time.Millisecond * 200 + + MemcachedInstance = "memcached" ) func TestAPIs(t *testing.T) { @@ -123,12 +126,16 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = mariadbv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = keystonev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) err = corev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) err = rabbitmqv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) err = keystonev1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = memcachedv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) logger = ctrl.Log.WithName("---Test---") //+kubebuilder:scaffold:scheme diff --git a/tests/functional/watcher_test_data.go b/tests/functional/watcher_test_data.go index 18a5666..d04950b 100644 --- a/tests/functional/watcher_test_data.go +++ b/tests/functional/watcher_test_data.go @@ -40,6 +40,8 @@ type WatcherTestData struct { InternalTopLevelSecretName types.NamespacedName WatcherTransportURL types.NamespacedName KeystoneServiceName types.NamespacedName + WatcherAPI types.NamespacedName + MemcachedNamespace types.NamespacedName } // GetWatcherTestData is a function that initialize the WatcherTestData @@ -80,5 +82,13 @@ func GetWatcherTestData(watcherName types.NamespacedName) WatcherTestData { Namespace: watcherName.Namespace, Name: "watcher", }, + WatcherAPI: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "watcher-api", + }, + MemcachedNamespace: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "memcached", + }, } } diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go index e1ef701..332fa95 100644 --- a/tests/functional/watcherapi_controller_test.go +++ b/tests/functional/watcherapi_controller_test.go @@ -7,37 +7,41 @@ import ( . "github.com/onsi/gomega" //revive:disable:dot-imports //revive:disable-next-line:dot-imports + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" "github.com/openstack-k8s-operators/watcher-operator/pkg/watcher" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" ) var ( MinimalWatcherAPISpec = map[string]interface{}{ - "secret": "osp-secret", - "databaseInstance": "openstack", + "secret": "osp-secret", + "databaseInstance": "openstack", + "memcachedInstance": "memcached", } ) var _ = Describe("WatcherAPI controller with minimal spec values", func() { When("A Watcher instance is created from minimal spec", func() { BeforeEach(func() { - DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, MinimalWatcherAPISpec)) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, MinimalWatcherAPISpec)) }) It("should have the Spec fields defaulted", func() { - WatcherAPI := GetWatcherAPI(watcherTest.Instance) + WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) Expect(WatcherAPI.Spec.DatabaseInstance).Should(Equal("openstack")) Expect(WatcherAPI.Spec.DatabaseAccount).Should(Equal("watcher")) Expect(WatcherAPI.Spec.Secret).Should(Equal("osp-secret")) + Expect(WatcherAPI.Spec.MemcachedInstance).Should(Equal("memcached")) Expect(WatcherAPI.Spec.PasswordSelectors).Should(Equal(watcherv1beta1.PasswordSelector{Service: "WatcherPassword"})) }) It("should have the Status fields initialized", func() { - WatcherAPI := GetWatcherAPI(watcherTest.Instance) + WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) Expect(WatcherAPI.Status.ObservedGeneration).To(Equal(int64(0))) }) @@ -45,7 +49,7 @@ var _ = Describe("WatcherAPI controller with minimal spec values", func() { // the reconciler loop adds the finalizer so we have to wait for // it to run Eventually(func() []string { - return GetWatcherAPI(watcherTest.Instance).Finalizers + return GetWatcherAPI(watcherTest.WatcherAPI).Finalizers }, timeout, interval).Should(ContainElement("openstack.org/watcherapi")) }) @@ -55,24 +59,25 @@ var _ = Describe("WatcherAPI controller with minimal spec values", func() { var _ = Describe("WatcherAPI controller", func() { When("A WatcherAPI instance is created", func() { BeforeEach(func() { - DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherAPISpec())) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) It("should have the Spec fields defaulted", func() { - WatcherAPI := GetWatcherAPI(watcherTest.Instance) + WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) Expect(WatcherAPI.Spec.DatabaseInstance).Should(Equal("openstack")) Expect(WatcherAPI.Spec.DatabaseAccount).Should(Equal("watcher")) Expect(WatcherAPI.Spec.Secret).Should(Equal("test-osp-secret")) + Expect(WatcherAPI.Spec.MemcachedInstance).Should(Equal("memcached")) }) It("should have the Status fields initialized", func() { - WatcherAPI := GetWatcherAPI(watcherTest.Instance) + WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) Expect(WatcherAPI.Status.ObservedGeneration).To(Equal(int64(0))) }) It("should have ReadyCondition false", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.ReadyCondition, corev1.ConditionFalse, @@ -81,7 +86,7 @@ var _ = Describe("WatcherAPI controller", func() { It("should have input not ready", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.InputReadyCondition, corev1.ConditionFalse, @@ -90,7 +95,7 @@ var _ = Describe("WatcherAPI controller", func() { It("should have service config input unknown", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.ServiceConfigReadyCondition, corev1.ConditionUnknown, @@ -101,16 +106,17 @@ var _ = Describe("WatcherAPI controller", func() { // the reconciler loop adds the finalizer so we have to wait for // it to run Eventually(func() []string { - return GetWatcherAPI(watcherTest.Instance).Finalizers + return GetWatcherAPI(watcherTest.WatcherAPI).Finalizers }, timeout, interval).Should(ContainElement("openstack.org/watcherapi")) }) }) - When("the secret is created with all the expected fields", func() { + When("the secret is created with all the expected fields and has all the required infra", func() { BeforeEach(func() { secret := th.CreateSecret( watcherTest.InternalTopLevelSecretName, map[string][]byte{ "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), }, ) DeferCleanup(k8sClient.Delete, ctx, secret) @@ -122,19 +128,35 @@ var _ = Describe("WatcherAPI controller", func() { watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) - DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherAPISpec())) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) + memcachedSpec := memcachedv1.MemcachedSpec{ + MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ + Replicas: ptr.To(int32(1)), + }, + } + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.WatcherAPI.Namespace, MemcachedInstance, memcachedSpec)) + infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) }) It("should have input ready", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.InputReadyCondition, corev1.ConditionTrue, ) }) + It("should have memcached ready true", func() { + th.ExpectCondition( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.MemcachedReadyCondition, + corev1.ConditionTrue, + ) + }) It("should have config service input ready", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.ServiceConfigReadyCondition, corev1.ConditionTrue, @@ -156,7 +178,7 @@ var _ = Describe("WatcherAPI controller", func() { watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) - DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherAPISpec())) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) It("should have input false", func() { errorString := fmt.Sprintf( @@ -164,7 +186,7 @@ var _ = Describe("WatcherAPI controller", func() { "field 'WatcherPassword' not found in secret/test-osp-secret", ) th.ExpectConditionWithDetails( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.InputReadyCondition, corev1.ConditionFalse, @@ -174,7 +196,7 @@ var _ = Describe("WatcherAPI controller", func() { }) It("should have config service input unknown", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.ServiceConfigReadyCondition, corev1.ConditionUnknown, @@ -183,11 +205,11 @@ var _ = Describe("WatcherAPI controller", func() { }) When("A WatcherAPI instance without secret is created", func() { BeforeEach(func() { - DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherAPISpec())) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) It("is missing the secret", func() { th.ExpectConditionWithDetails( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.InputReadyCondition, corev1.ConditionFalse, @@ -202,13 +224,14 @@ var _ = Describe("WatcherAPI controller", func() { watcherTest.InternalTopLevelSecretName, map[string][]byte{ "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), }, ) DeferCleanup(k8sClient.Delete, ctx, secret) - DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.Instance, GetDefaultWatcherAPISpec())) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) It("should have input not ready", func() { - WatcherAPI := GetWatcherAPI(watcherTest.Instance) + WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) customErrorString := fmt.Sprintf( "couldn't get database %s and account %s", watcher.DatabaseCRName, @@ -219,7 +242,7 @@ var _ = Describe("WatcherAPI controller", func() { customErrorString, ) th.ExpectConditionWithDetails( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.InputReadyCondition, corev1.ConditionFalse, @@ -229,11 +252,109 @@ var _ = Describe("WatcherAPI controller", func() { }) It("should have config service unknown", func() { th.ExpectCondition( - watcherTest.Instance, + watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), condition.ServiceConfigReadyCondition, corev1.ConditionUnknown, ) }) }) + When("secret and db are created, but there is no memcached", func() { + BeforeEach(func() { + secret := th.CreateSecret( + watcherTest.InternalTopLevelSecretName, + map[string][]byte{ + "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, secret) + mariadb.CreateMariaDBDatabase(watcherTest.WatcherDatabaseName.Namespace, watcherTest.WatcherDatabaseName.Name, mariadbv1.MariaDBDatabaseSpec{}) + DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(watcherTest.WatcherDatabaseName)) + + mariadb.SimulateMariaDBTLSDatabaseCompleted(watcherTest.WatcherDatabaseName) + apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret( + watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) + DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) + }) + It("should have input ready true", func() { + th.ExpectCondition( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.InputReadyCondition, + corev1.ConditionTrue, + ) + }) + It("should have memcached ready false", func() { + th.ExpectConditionWithDetails( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.MemcachedReadyCondition, + corev1.ConditionFalse, + condition.RequestedReason, + condition.MemcachedReadyWaitingMessage, + ) + }) + }) + When("secret, db and memcached are created, but there is no keystoneapi", func() { + BeforeEach(func() { + secret := th.CreateSecret( + watcherTest.InternalTopLevelSecretName, + map[string][]byte{ + "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, secret) + mariadb.CreateMariaDBDatabase(watcherTest.WatcherDatabaseName.Namespace, watcherTest.WatcherDatabaseName.Name, mariadbv1.MariaDBDatabaseSpec{}) + DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(watcherTest.WatcherDatabaseName)) + + mariadb.SimulateMariaDBTLSDatabaseCompleted(watcherTest.WatcherDatabaseName) + apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret( + watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) + DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) + DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) + memcachedSpec := memcachedv1.MemcachedSpec{ + MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ + Replicas: ptr.To(int32(1)), + }, + } + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.WatcherAPI.Namespace, MemcachedInstance, memcachedSpec)) + infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) + + }) + It("should have input ready true", func() { + th.ExpectCondition( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.InputReadyCondition, + corev1.ConditionTrue, + ) + }) + It("should have memcached ready true", func() { + th.ExpectCondition( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.MemcachedReadyCondition, + corev1.ConditionTrue, + ) + }) + It("should have config service input unknown", func() { + errorString := fmt.Sprintf( + condition.ServiceConfigReadyErrorMessage, + "keystoneAPI not found", + ) + th.ExpectConditionWithDetails( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.ServiceConfigReadyCondition, + corev1.ConditionFalse, + condition.ErrorReason, + errorString, + ) + }) + }) }) diff --git a/tests/kuttl/test-suites/default/watcher-api/03-assert.yaml b/tests/kuttl/test-suites/default/watcher-api/03-assert.yaml index 84ab971..a524eee 100644 --- a/tests/kuttl/test-suites/default/watcher-api/03-assert.yaml +++ b/tests/kuttl/test-suites/default/watcher-api/03-assert.yaml @@ -20,7 +20,17 @@ status: reason: Ready status: "True" type: InputReady + - message: " Memcached instance has been provisioned" + reason: Ready + status: "True" + type: MemcachedReady - message: Service config create completed reason: Ready status: "True" type: ServiceConfigReady +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcherapi-kuttl-config-data +type: Opaque diff --git a/tests/kuttl/test-suites/default/watcher-api/03-deploy-watcher-api.yaml b/tests/kuttl/test-suites/default/watcher-api/03-deploy-watcher-api.yaml index 38e6684..1e0af9c 100644 --- a/tests/kuttl/test-suites/default/watcher-api/03-deploy-watcher-api.yaml +++ b/tests/kuttl/test-suites/default/watcher-api/03-deploy-watcher-api.yaml @@ -5,6 +5,7 @@ metadata: type: Opaque stringData: WatcherPassword: password + transport_url: rabbitmq-transport-url-watcher-kuttl-watcher-transport --- apiVersion: watcher.openstack.org/v1beta1 kind: WatcherAPI @@ -13,3 +14,4 @@ metadata: spec: databaseInstance: openstack secret: watcherapi-secret + memcachedInstance: "memcached"