From b863d63105e994ad0d30c7cc2eb67b4acf44a1fd Mon Sep 17 00:00:00 2001 From: Kaustav Majumder Date: Thu, 3 Oct 2024 16:17:38 +0530 Subject: [PATCH] Create backingStore for each storageconsumer Signed-off-by: Kaustav Majumder --- config/rbac/role.yaml | 1 + controllers/storageconsumer/consumer_test.go | 14 +++++ .../storageconsumer_controller.go | 58 ++++++++++++++++++- deploy/csv-templates/ocs-operator.csv.yaml.in | 1 + .../ocs-operator.clusterserviceversion.yaml | 1 + services/provider/server/consumer_test.go | 6 +- services/provider/server/server.go | 33 ++++++----- services/provider/server/server_test.go | 28 +++++---- 8 files changed, 110 insertions(+), 32 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 96b5519fbc..df51fe649a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -166,6 +166,7 @@ rules: - apiGroups: - noobaa.io resources: + - backingstores - noobaaaccounts - noobaas verbs: diff --git a/controllers/storageconsumer/consumer_test.go b/controllers/storageconsumer/consumer_test.go index 0cdeb8310e..8d4bf333b9 100644 --- a/controllers/storageconsumer/consumer_test.go +++ b/controllers/storageconsumer/consumer_test.go @@ -22,6 +22,7 @@ import ( noobaaApis "github.com/noobaa/noobaa-operator/v5/pkg/apis" "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" v1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" @@ -128,6 +129,14 @@ func TestCephName(t *testing.T) { Phase: v1alpha1.NooBaaAccountPhaseReady, }, } + r.backingStore = &nbv1.BackingStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "consumer-acc", + }, + Status: nbv1.BackingStoreStatus{ + Phase: nbv1.BackingStorePhaseReady, + }, + } clusterVersionProvider := &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ Name: "version", @@ -152,6 +161,11 @@ func TestCephName(t *testing.T) { Name: "consumer-acc", Phase: "Ready", }, + { + Kind: "BackingStore", + Name: "consumer-acc", + Phase: "Ready", + }, } assert.Equal(t, r.storageConsumer.Status.CephResources, want) diff --git a/controllers/storageconsumer/storageconsumer_controller.go b/controllers/storageconsumer/storageconsumer_controller.go index 2a4535a2c2..cec2f419e6 100644 --- a/controllers/storageconsumer/storageconsumer_controller.go +++ b/controllers/storageconsumer/storageconsumer_controller.go @@ -28,7 +28,9 @@ import ( "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -64,6 +66,7 @@ type StorageConsumerReconciler struct { cephClientHealthChecker *rookCephv1.CephClient namespace string noobaaAccount *nbv1.NooBaaAccount + backingStore *nbv1.BackingStore } //+kubebuilder:rbac:groups=ocs.openshift.io,resources=storageconsumers,verbs=get;list;watch;create;update;patch;delete @@ -71,6 +74,7 @@ type StorageConsumerReconciler struct { //+kubebuilder:rbac:groups=ocs.openshift.io,resources=storageconsumers/status,verbs=get;update;patch // +kubebuilder:rbac:groups=ocs.openshift.io,resources=storagerequests,verbs=get;list; // +kubebuilder:rbac:groups=noobaa.io,resources=noobaaaccounts,verbs=get;list;watch;create;update;delete +// +kubebuilder:rbac:groups=noobaa.io,resources=backingstores,verbs=get;list;watch;create;update;delete // Reconcile reads that state of the cluster for a StorageConsumer object and makes changes based on the state read // and what is in the StorageConsumer.Spec @@ -137,6 +141,10 @@ func (r *StorageConsumerReconciler) initReconciler(request reconcile.Request) { r.noobaaAccount = &nbv1.NooBaaAccount{} r.noobaaAccount.Name = r.storageConsumer.Name r.noobaaAccount.Namespace = r.storageConsumer.Namespace + + r.backingStore = &nbv1.BackingStore{} + r.backingStore.Name = r.storageConsumer.Name + r.backingStore.Namespace = r.storageConsumer.Namespace } func (r *StorageConsumerReconciler) reconcilePhases() (reconcile.Result, error) { @@ -159,6 +167,9 @@ func (r *StorageConsumerReconciler) reconcilePhases() (reconcile.Result, error) // A NooBaa account only needs to be created if the storage consumer is for a client cluster. clusterID := util.GetClusterID(r.ctx, r.Client, &r.Log) if clusterID != "" && !strings.Contains(r.storageConsumer.Name, clusterID) { + if err := r.reconcileRemoteBackingStore(); err != nil { + return reconcile.Result{}, err + } if err := r.reconcileNoobaaAccount(); err != nil { return reconcile.Result{}, err } @@ -224,13 +235,54 @@ func (r *StorageConsumerReconciler) reconcileCephClientHealthChecker() error { return nil } +func (r *StorageConsumerReconciler) reconcileRemoteBackingStore() error { + + _, err := ctrl.CreateOrUpdate(r.ctx, r.Client, r.backingStore, func() error { + if err := r.own(r.backingStore); err != nil { + return err + } + r.backingStore = &nbv1.BackingStore{ + Spec: nbv1.BackingStoreSpec{ + Type: nbv1.StoreTypePVPool, + PVPool: &nbv1.PVPoolSpec{ + // TODO: query to get storageclass + StorageClass: "ocs-storagecluster-ceph-rbd", + NumVolumes: 1, + VolumeResources: &v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("50Gi"), + }, + }, + }, + }, + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create noobaa backingstore for storageConsumer %v: %v", r.storageConsumer.Name, err) + } + + phase := string(r.backingStore.Status.Phase) + r.setCephResourceStatus(r.backingStore.Name, "BackingStore", phase, nil) + + return nil +} + func (r *StorageConsumerReconciler) reconcileNoobaaAccount() error { + + // if err := r.get(r.backingStore); err != nil { + // return err + // } + + // if r.backingStore.Status.Phase != nbv1.BackingStorePhaseReady { + // return fmt.Errorf("noobaa backing store with name %v is not yet ready ", r.backingStore.Name) + // } _, err := ctrl.CreateOrUpdate(r.ctx, r.Client, r.noobaaAccount, func() error { if err := r.own(r.noobaaAccount); err != nil { return err } - // TODO: query the name of backing store during runtime - r.noobaaAccount.Spec.DefaultResource = "noobaa-default-backing-store" + r.noobaaAccount.Spec.AllowBucketCreate = true + r.noobaaAccount.Spec.DefaultResource = r.backingStore.Name // the following annotation will enable noobaa-operator to create a auth_token secret based on this account util.AddAnnotation(r.noobaaAccount, "remote-operator", "true") return nil @@ -296,6 +348,8 @@ func (r *StorageConsumerReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&rookCephv1.CephBlockPool{}, enqueueStorageConsumerRequest, ). + Owns(&nbv1.NooBaaAccount{}). + Owns(&nbv1.BackingStore{}). Complete(r) } diff --git a/deploy/csv-templates/ocs-operator.csv.yaml.in b/deploy/csv-templates/ocs-operator.csv.yaml.in index a230eb4102..958e56a89b 100644 --- a/deploy/csv-templates/ocs-operator.csv.yaml.in +++ b/deploy/csv-templates/ocs-operator.csv.yaml.in @@ -337,6 +337,7 @@ spec: - apiGroups: - noobaa.io resources: + - backingstores - noobaaaccounts - noobaas verbs: diff --git a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml index ecb465f9b9..c61ab71084 100644 --- a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml +++ b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml @@ -346,6 +346,7 @@ spec: - apiGroups: - noobaa.io resources: + - backingstores - noobaaaccounts - noobaas verbs: diff --git a/services/provider/server/consumer_test.go b/services/provider/server/consumer_test.go index 9d378eeb54..a07d4d344b 100644 --- a/services/provider/server/consumer_test.go +++ b/services/provider/server/consumer_test.go @@ -4,7 +4,7 @@ import ( "context" "testing" - routev1 "github.com/openshift/api/route/v1" + nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" api "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -67,8 +67,8 @@ func newFakeClient(t *testing.T, obj ...client.Object) client.Client { err = opv1a1.AddToScheme(scheme) assert.NoError(t, err, "failed to add opv1a1 scheme") - err = routev1.AddToScheme(scheme) - assert.NoError(t, err, "failed to add routev1 scheme") + err = nbapis.AddToScheme(scheme) + assert.NoError(t, err, "failed to add nbapis scheme") return fake.NewClientBuilder(). WithScheme(scheme). WithObjects(obj...). diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 9f127f80d5..df5e0db171 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -11,7 +11,6 @@ import ( "encoding/json" "encoding/pem" "fmt" - "k8s.io/utils/ptr" "math" "net" "slices" @@ -19,10 +18,12 @@ import ( "strings" "time" + "k8s.io/utils/ptr" + "github.com/blang/semver/v4" + nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis" nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" - routev1 "github.com/openshift/api/route/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -261,9 +262,9 @@ func newClient() (client.Client, error) { if err != nil { return nil, fmt.Errorf("failed to add ocsv1 to scheme. %v", err) } - err = routev1.AddToScheme(scheme) + err = nbapis.AddToScheme(scheme) if err != nil { - return nil, fmt.Errorf("failed to add routev1 to scheme. %v", err) + return nil, fmt.Errorf("failed to add nbapis to scheme. %v", err) } config, err := config.GetConfig() @@ -413,21 +414,21 @@ func (s *OCSProviderServer) getExternalResources(ctx context.Context, consumerRe return nil, fmt.Errorf("auth_token not found in %s secret", noobaaOperatorSecret.Name) } - noobaMgmtRoute := &routev1.Route{} - noobaMgmtRoute.Name = "noobaa-mgmt" - noobaMgmtRoute.Namespace = s.namespace + nb := &nbv1.NooBaa{} + nb.Name = "noobaa" + nb.Namespace = s.namespace - if err = s.client.Get(ctx, client.ObjectKeyFromObject(noobaMgmtRoute), noobaMgmtRoute); err != nil { - return nil, fmt.Errorf("failed to get noobaa-mgmt route. %v", err) - } - if noobaMgmtRoute.Status.Ingress == nil || len(noobaMgmtRoute.Status.Ingress) == 0 { - return nil, fmt.Errorf("no Ingress available in noobaa-mgmt route") + if err = s.client.Get(ctx, client.ObjectKeyFromObject(nb), nb); err != nil { + return nil, fmt.Errorf("failed to get noobaa %v", err) } - noobaaMgmtAddress := noobaMgmtRoute.Status.Ingress[0].Host - if noobaaMgmtAddress == "" { - return nil, fmt.Errorf("no Host found in noobaa-mgmt route Ingress") + if nb.Status.Phase != nbv1.SystemPhaseReady { + // Noobaa system is not yet ready + return extR, nil } + + noobaaMgmtAddress := nb.Status.Services.ServiceMgmt.ExternalDNS[0] + extR = append(extR, &pb.ExternalResource{ Name: "noobaa-remote-join-secret", Kind: "Secret", @@ -438,7 +439,7 @@ func (s *OCSProviderServer) getExternalResources(ctx context.Context, consumerRe }) extR = append(extR, &pb.ExternalResource{ - Name: "noobaa-remote", + Name: "noobaa", Kind: "Noobaa", Data: mustMarshal(&nbv1.NooBaaSpec{ JoinSecret: &v1.SecretReference{ diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index e23635362f..fdcf1b2816 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -10,7 +10,6 @@ import ( csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" - routev1 "github.com/openshift/api/route/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -58,6 +57,15 @@ var ocsSubscriptionSpec = &opv1a1.SubscriptionSpec{ Channel: "1.0", Package: "ocs-operator", } +var nbProviderStatus = &nbv1.NooBaaStatus{ + Phase: nbv1.SystemPhaseReady, + Services: &nbv1.ServicesStatus{ + ServiceMgmt: nbv1.ServiceStatus{ + ExternalDNS: []string{"noobaaMgmtAddress"}, + }, + }, +} + var noobaaSpec = &nbv1.NooBaaSpec{ JoinSecret: &v1.SecretReference{ Name: "noobaa-remote-join-secret", @@ -100,8 +108,8 @@ var mockExtR = map[string]*externalResource{ Kind: "Secret", Data: joinSecret, }, - "noobaa-remote": { - Name: "noobaa-remote", + "noobaa": { + Name: "noobaa", Kind: "Noobaa", Data: noobaaSpec, }, @@ -254,16 +262,8 @@ func TestGetExternalResources(t *testing.T) { }, } - noobaaMgmtRoute := &routev1.Route{ - ObjectMeta: metav1.ObjectMeta{Name: "noobaa-mgmt", Namespace: server.namespace}, - Status: routev1.RouteStatus{ - Ingress: []routev1.RouteIngress{{Host: "noobaaMgmtAddress"}}, - }, - } - assert.NoError(t, client.Create(ctx, noobaaRemoteJoinSecretConsumer)) assert.NoError(t, client.Create(ctx, noobaaRemoteJoinSecretConsumer6)) - assert.NoError(t, client.Create(ctx, noobaaMgmtRoute)) monCm, monSc := createMonConfigMapAndSecret(server) assert.NoError(t, client.Create(ctx, monCm)) @@ -275,6 +275,12 @@ func TestGetExternalResources(t *testing.T) { ocsSubscription.Spec = ocsSubscriptionSpec assert.NoError(t, client.Create(ctx, ocsSubscription)) + noobaa := &nbv1.NooBaa{} + noobaa.Name = "noobaa" + noobaa.Namespace = serverNamespace + noobaa.Status = *nbProviderStatus + assert.NoError(t, client.Create(ctx, noobaa)) + // When ocsv1alpha1.StorageConsumerStateReady req := pb.StorageConfigRequest{ StorageConsumerUUID: string(consumerResource.UID),