add server implementation for peerblockpool rpc call

Signed-off-by: Rewant Soni <[email protected]>

deploy/ocs-operator/manifests/provider-role.yaml

@@ -7,10 +7,23 @@ rules:
   - ""
   resources:
   - configmaps
-  - secrets
   - services
   verbs:
   - get
+- apiGroups:
+    - ""
+  resources:
+    - secrets
+  verbs:
+    - get
+    - create
+    - update
+- apiGroups:
+    - ceph.rook.io
+  resources:
+    - cephblockpools/finalizers
+  verbs:
+    - update
 - apiGroups:
   - ceph.rook.io
   resources:
@@ -19,6 +32,13 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+    - ceph.rook.io
+  resources:
+    - cephblockpools
+  verbs:
+    - get
+    - update
 - apiGroups:
   - ocs.openshift.io
   resources:
@@ -61,3 +81,10 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+    - ceph.rook.io
+  resources:
+    - cephrbdmirrors
+  verbs:
+    - get
+    - create

rbac/provider-role.yaml

@@ -7,10 +7,23 @@ rules:
   - ""
   resources:
   - configmaps
-  - secrets
   - services
   verbs:
   - get
+- apiGroups:
+    - ""
+  resources:
+    - secrets
+  verbs:
+    - get
+    - create
+    - update
+- apiGroups:
+    - ceph.rook.io
+  resources:
+    - cephblockpools/finalizers
+  verbs:
+    - update
 - apiGroups:
   - ceph.rook.io
   resources:
@@ -19,6 +32,13 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+    - ceph.rook.io
+  resources:
+    - cephblockpools
+  verbs:
+    - get
+    - update
 - apiGroups:
   - ocs.openshift.io
   resources:
@@ -61,3 +81,10 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+    - ceph.rook.io
+  resources:
+    - cephrbdmirrors
+  verbs:
+    - get
+    - create

services/provider/server/cephblockpool.go

@@ -0,0 +1,85 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"slices"
+
+	rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	corev1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type cephBlockPoolManager struct {
+	client    client.Client
+	namespace string
+}
+
+func newCephBlockPoolManager(cl client.Client, namespace string) (*cephBlockPoolManager, error) {
+	return &cephBlockPoolManager{
+		client:    cl,
+		namespace: namespace,
+	}, nil
+}
+
+func (c *cephBlockPoolManager) EnableBlockPoolMirroring(ctx context.Context, cephBlockPool *rookCephv1.CephBlockPool) error {
+
+	cephBlockPool.Spec.Mirroring.Enabled = true
+	cephBlockPool.Spec.Mirroring.Mode = "image"
+
+	err := c.client.Update(ctx, cephBlockPool)
+	if err != nil {
+		return fmt.Errorf("failed to enable mirroring on CephBlockPool resource with name %q: %v", cephBlockPool.Name, err)
+	}
+
+	return nil
+
+}
+
+func (c *cephBlockPoolManager) SetBootstrapSecretRef(ctx context.Context, cephBlockPool *rookCephv1.CephBlockPool, secretName string, secretData map[string][]byte) error {
+
+	// create the secret
+	bootstrapSecret := &corev1.Secret{}
+	bootstrapSecret.Name = secretName
+	bootstrapSecret.Namespace = c.namespace
+
+	_, err := ctrl.CreateOrUpdate(ctx, c.client, bootstrapSecret, func() error {
+		bootstrapSecret.Data = secretData
+		return ctrl.SetControllerReference(cephBlockPool, bootstrapSecret, c.client.Scheme())
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create/update the bootstrap secret %q: %v", secretName, err)
+	}
+
+	// set the secret ref
+	if cephBlockPool.Spec.Mirroring.Peers == nil {
+		cephBlockPool.Spec.Mirroring.Peers = &rookCephv1.MirroringPeerSpec{SecretNames: []string{secretName}}
+	} else {
+		if !slices.Contains(cephBlockPool.Spec.Mirroring.Peers.SecretNames, secretName) {
+			cephBlockPool.Spec.Mirroring.Peers.SecretNames = append(cephBlockPool.Spec.Mirroring.Peers.SecretNames, secretName)
+		}
+	}
+
+	err = c.client.Update(ctx, cephBlockPool)
+	if err != nil {
+		return fmt.Errorf("failed to set bootstrap secret ref on CephBlockPool resource with name %q: %v", cephBlockPool.Name, err)
+	}
+
+	return nil
+}
+
+func (c *cephBlockPoolManager) GetBlockPoolByName(ctx context.Context, blockPoolName string) (*rookCephv1.CephBlockPool, error) {
+	blockPool := &rookCephv1.CephBlockPool{}
+	blockPool.Name = blockPoolName
+	blockPool.Namespace = c.namespace
+	err := c.client.Get(ctx, client.ObjectKeyFromObject(blockPool), blockPool)
+	if err != nil {
+		if kerrors.IsNotFound(err) {
+			return nil, fmt.Errorf("CephBlockPool resource %q not found: %v", blockPoolName, err)
+		}
+		return nil, fmt.Errorf("failed to get CephBlockPool resource with name %q: %v", blockPoolName, err)
+	}
+	return blockPool, nil
+}

services/provider/server/cephrbdmirror.go

@@ -0,0 +1,43 @@
+package server
+
+import (
+	"context"
+
+	rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const rBDMirrorName = "rbd-mirror"
+
+type cephRBDMirrorManager struct {
+	client    client.Client
+	namespace string
+}
+
+func newCephRBDMirrorManager(cl client.Client, namespace string) (*cephRBDMirrorManager, error) {
+	return &cephRBDMirrorManager{
+		client:    cl,
+		namespace: namespace,
+	}, nil
+}
+
+func (c *cephRBDMirrorManager) Create(ctx context.Context) error {
+
+	cephRBDMirror := &rookCephv1.CephRBDMirror{}
+	cephRBDMirror.Name = rBDMirrorName
+	cephRBDMirror.Namespace = c.namespace
+	err := c.client.Get(ctx, client.ObjectKeyFromObject(cephRBDMirror), cephRBDMirror)
+
+	// create if not found
+	if err != nil && kerrors.IsNotFound(err) {
+		cephRBDMirror.Spec = rookCephv1.RBDMirroringSpec{Count: 1}
+		err = c.client.Create(ctx, cephRBDMirror)
+		if err != nil {
+			return err
+		}
+	}
+
+	// if any other err/nil return it
+	return err
+}

services/provider/server/server.go

@@ -60,6 +60,8 @@ type OCSProviderServer struct {
 	client                     client.Client
 	consumerManager            *ocsConsumerManager
 	storageClassRequestManager *storageClassRequestManager
+	cephBlockPoolManager       *cephBlockPoolManager
+	cephRBDMirrorManager       *cephRBDMirrorManager
 	namespace                  string
 }
 
@@ -79,10 +81,23 @@ func NewOCSProviderServer(ctx context.Context, namespace string) (*OCSProviderSe
 		return nil, fmt.Errorf("failed to create new StorageClassRequest instance. %v", err)
 	}
 
+	cephBlockPoolManager, err := newCephBlockPoolManager(client, namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new CephBlockPool instance. %v", err)
+	}
+
+	cephRBDMirrorManager, err := newCephRBDMirrorManager(client, namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create new CephRBDMirror instance. %v", err)
+
+	}
+
 	return &OCSProviderServer{
 		client:                     client,
 		consumerManager:            consumerManager,
 		storageClassRequestManager: storageClassRequestManager,
+		cephBlockPoolManager:       cephBlockPoolManager,
+		cephRBDMirrorManager:       cephRBDMirrorManager,
 		namespace:                  namespace,
 	}, nil
 }
@@ -733,3 +748,34 @@ func (s *OCSProviderServer) getOCSSubscriptionChannel(ctx context.Context) (stri
 	}
 	return subscription.Spec.Channel, nil
 }
+
+// PeerBlockPool RPC call to send the bootstrap secret for the pool
+func (s *OCSProviderServer) PeerBlockPool(ctx context.Context, req *pb.PeerBlockPoolRequest) (*pb.PeerBlockPoolResponse, error) {
+
+	cephBlockPool, err := s.cephBlockPoolManager.GetBlockPoolByName(ctx, string(req.Pool))
+	if err != nil {
+		return nil, status.Errorf(codes.NotFound, "Failed to find CephBlockPool resource %s: %v", req.Pool, err)
+	}
+
+	if err := s.cephRBDMirrorManager.Create(ctx); err != nil {
+		return nil, status.Errorf(codes.Internal, "Failed to create RBDMirror instance: %v", err)
+	}
+
+	// enable mirroring on blockPool in the req
+	if err := s.cephBlockPoolManager.EnableBlockPoolMirroring(ctx, cephBlockPool); err != nil {
+		return nil, status.Errorf(codes.Internal, "Failed to enable mirroring for CephBlockPool resource %s: %v", req.Pool, err)
+	}
+
+	// create and set secret ref on the blockPool
+	if err := s.cephBlockPoolManager.SetBootstrapSecretRef(
+		ctx,
+		cephBlockPool,
+		req.SecretName,
+		map[string][]byte{
+			"pool":  req.Pool,
+			"token": req.Token,
+		}); err != nil {
+		return nil, status.Errorf(codes.Internal, "Failed to set bootstrap secret ref for CephBlockPool resource %s: %v", req.Pool, err)
+	}
+	return &pb.PeerBlockPoolResponse{}, nil
+}