From d861d4a1719378d218cf5fcf073a27efbfce0ce9 Mon Sep 17 00:00:00 2001 From: Rewant Soni Date: Mon, 26 Feb 2024 18:37:06 +0530 Subject: [PATCH] add server implementation for peerblockpool rpc call Signed-off-by: Rewant Soni --- .../ocs-operator/manifests/provider-role.yaml | 30 ++++++- rbac/provider-role.yaml | 30 ++++++- services/provider/server/cephblockpool.go | 85 +++++++++++++++++++ services/provider/server/cephrbdmirror.go | 43 ++++++++++ services/provider/server/server.go | 46 ++++++++++ 5 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 services/provider/server/cephblockpool.go create mode 100644 services/provider/server/cephrbdmirror.go diff --git a/deploy/ocs-operator/manifests/provider-role.yaml b/deploy/ocs-operator/manifests/provider-role.yaml index 26d6589e50..1ba8197f50 100644 --- a/deploy/ocs-operator/manifests/provider-role.yaml +++ b/deploy/ocs-operator/manifests/provider-role.yaml @@ -7,10 +7,23 @@ rules: - "" resources: - configmaps - - secrets - services verbs: - get +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools/finalizers + verbs: + - update - apiGroups: - ceph.rook.io resources: @@ -18,6 +31,14 @@ rules: verbs: - get - list +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools + verbs: + - get + - update + - create - apiGroups: - ocs.openshift.io resources: @@ -53,3 +74,10 @@ rules: - list - create - delete +- apiGroups: + - ceph.rook.io + resources: + - cephrbdmirrors + verbs: + - get + - create diff --git a/rbac/provider-role.yaml b/rbac/provider-role.yaml index 26d6589e50..1ba8197f50 100644 --- a/rbac/provider-role.yaml +++ b/rbac/provider-role.yaml @@ -7,10 +7,23 @@ rules: - "" resources: - configmaps - - secrets - services verbs: - get +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools/finalizers + verbs: + - update - apiGroups: - ceph.rook.io resources: @@ -18,6 +31,14 @@ rules: verbs: - get - list +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools + verbs: + - get + - update + - create - apiGroups: - ocs.openshift.io resources: @@ -53,3 +74,10 @@ rules: - list - create - delete +- apiGroups: + - ceph.rook.io + resources: + - cephrbdmirrors + verbs: + - get + - create diff --git a/services/provider/server/cephblockpool.go b/services/provider/server/cephblockpool.go new file mode 100644 index 0000000000..f7fa79b20b --- /dev/null +++ b/services/provider/server/cephblockpool.go @@ -0,0 +1,85 @@ +package server + +import ( + "context" + "fmt" + "slices" + + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type cephBlockPoolManager struct { + client client.Client + namespace string +} + +func newCephBlockPoolManager(cl client.Client, namespace string) (*cephBlockPoolManager, error) { + return &cephBlockPoolManager{ + client: cl, + namespace: namespace, + }, nil +} + +func (c *cephBlockPoolManager) EnableBlockPoolMirroring(ctx context.Context, cephBlockPool *rookCephv1.CephBlockPool) error { + + cephBlockPool.Spec.Mirroring.Enabled = true + cephBlockPool.Spec.Mirroring.Mode = "image" + + err := c.client.Patch(ctx, cephBlockPool, client.MergeFrom(cephBlockPool)) + if err != nil { + return fmt.Errorf("failed to enable mirroring on CephBlockPool resource with name %q: %v", cephBlockPool.Name, err) + } + + return nil + +} + +func (c *cephBlockPoolManager) SetBootstrapSecretRef(ctx context.Context, cephBlockPool *rookCephv1.CephBlockPool, secretName string, secretData map[string][]byte) error { + + // create the secret + bootstrapSecret := &corev1.Secret{} + bootstrapSecret.Name = secretName + bootstrapSecret.Namespace = c.namespace + + _, err := ctrl.CreateOrUpdate(ctx, c.client, bootstrapSecret, func() error { + bootstrapSecret.Data = secretData + return ctrl.SetControllerReference(cephBlockPool, bootstrapSecret, c.client.Scheme()) + }) + if err != nil { + return fmt.Errorf("failed to create/update the bootstrap secret %q: %v", secretName, err) + } + + // set the secret ref + if cephBlockPool.Spec.Mirroring.Peers == nil { + cephBlockPool.Spec.Mirroring.Peers = &rookCephv1.MirroringPeerSpec{SecretNames: []string{secretName}} + } else { + if !slices.Contains(cephBlockPool.Spec.Mirroring.Peers.SecretNames, secretName) { + cephBlockPool.Spec.Mirroring.Peers.SecretNames = append(cephBlockPool.Spec.Mirroring.Peers.SecretNames, secretName) + } + } + + err = c.client.Patch(ctx, cephBlockPool, client.MergeFrom(cephBlockPool)) + if err != nil { + return fmt.Errorf("failed to set bootstrap secret ref on CephBlockPool resource with name %q: %v", cephBlockPool.Name, err) + } + + return nil +} + +func (c *cephBlockPoolManager) GetBlockPoolByName(ctx context.Context, blockPoolName string) (*rookCephv1.CephBlockPool, error) { + blockPool := &rookCephv1.CephBlockPool{} + blockPool.Name = blockPoolName + blockPool.Namespace = c.namespace + err := c.client.Get(ctx, client.ObjectKeyFromObject(blockPool), blockPool) + if err != nil { + if kerrors.IsNotFound(err) { + return nil, fmt.Errorf("CephBlockPool resource %q not found: %v", blockPoolName, err) + } + return nil, fmt.Errorf("failed to get CephBlockPool resource with name %q: %v", blockPoolName, err) + } + return blockPool, nil +} diff --git a/services/provider/server/cephrbdmirror.go b/services/provider/server/cephrbdmirror.go new file mode 100644 index 0000000000..ebafc0becd --- /dev/null +++ b/services/provider/server/cephrbdmirror.go @@ -0,0 +1,43 @@ +package server + +import ( + "context" + + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const rBDMirrorName = "rbd-mirror" + +type cephRBDMirrorManager struct { + client client.Client + namespace string +} + +func newCephRBDMirrorManager(cl client.Client, namespace string) (*cephRBDMirrorManager, error) { + return &cephRBDMirrorManager{ + client: cl, + namespace: namespace, + }, nil +} + +func (c *cephRBDMirrorManager) Create(ctx context.Context) error { + + cephRBDMirror := &rookCephv1.CephRBDMirror{} + cephRBDMirror.Name = rBDMirrorName + cephRBDMirror.Namespace = c.namespace + err := c.client.Get(ctx, client.ObjectKeyFromObject(cephRBDMirror), cephRBDMirror) + + // create if not found + if err != nil && kerrors.IsNotFound(err) { + cephRBDMirror.Spec = rookCephv1.RBDMirroringSpec{Count: 1} + err = c.client.Create(ctx, cephRBDMirror) + if err != nil { + return err + } + } + + // if any other err/nil return it + return err +} diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 28b63f0aff..2a3e1327c5 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -58,6 +58,8 @@ type OCSProviderServer struct { client client.Client consumerManager *ocsConsumerManager storageClassRequestManager *storageClassRequestManager + cephBlockPoolManager *cephBlockPoolManager + cephRBDMirrorManager *cephRBDMirrorManager namespace string } @@ -77,10 +79,23 @@ func NewOCSProviderServer(ctx context.Context, namespace string) (*OCSProviderSe return nil, fmt.Errorf("failed to create new StorageClassRequest instance. %v", err) } + cephBlockPoolManager, err := newCephBlockPoolManager(client, namespace) + if err != nil { + return nil, fmt.Errorf("failed to create new CephBlockPool instance. %v", err) + } + + cephRBDMirrorManager, err := newCephRBDMirrorManager(client, namespace) + if err != nil { + return nil, fmt.Errorf("failed to create new CephRBDMirror instance. %v", err) + + } + return &OCSProviderServer{ client: client, consumerManager: consumerManager, storageClassRequestManager: storageClassRequestManager, + cephBlockPoolManager: cephBlockPoolManager, + cephRBDMirrorManager: cephRBDMirrorManager, namespace: namespace, }, nil } @@ -690,3 +705,34 @@ func (s *OCSProviderServer) ReportStatus(ctx context.Context, req *pb.ReportStat return &pb.ReportStatusResponse{}, nil } + +// PeerBlockPool RPC call to send the bootstrap secret for the pool +func (s *OCSProviderServer) PeerBlockPool(ctx context.Context, req *pb.PeerBlockPoolRequest) (*pb.PeerBlockPoolResponse, error) { + + cephBlockPool, err := s.cephBlockPoolManager.GetBlockPoolByName(ctx, string(req.Pool)) + if err != nil { + return nil, status.Errorf(codes.NotFound, "Failed to find CephBlockPool resource %s: %v", req.Pool, err) + } + + if err := s.cephRBDMirrorManager.Create(ctx); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create RBDMirror instance: %v", err) + } + + // enable mirroring on blockPool in the req + if err := s.cephBlockPoolManager.EnableBlockPoolMirroring(ctx, cephBlockPool); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to enable mirroring for CephBlockPool resource %s: %v", req.Pool, err) + } + + // create and set secret ref on the blockPool + if err := s.cephBlockPoolManager.SetBootstrapSecretRef( + ctx, + cephBlockPool, + req.SecretName, + map[string][]byte{ + "pool": req.Pool, + "token": req.Token, + }); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to set bootstrap secret ref for CephBlockPool resource %s: %v", req.Pool, err) + } + return &pb.PeerBlockPoolResponse{}, nil +}