diff --git a/transfer/rsync/client.go b/transfer/rsync/client.go new file mode 100644 index 0000000..fe523dc --- /dev/null +++ b/transfer/rsync/client.go @@ -0,0 +1,476 @@ +package rsync + +import ( + "context" + "fmt" + "strings" + + "github.com/backube/pvc-transfer/endpoint" + "github.com/backube/pvc-transfer/internal/utils" + "github.com/backube/pvc-transfer/transfer" + "github.com/backube/pvc-transfer/transport" + "github.com/backube/pvc-transfer/transport/stunnel" + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + errorsutil "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/utils/pointer" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type client struct { + username string + password string + pvcList transfer.PVCList + transportClient transport.Transport + endpoint endpoint.Endpoint + + nameSuffix string + + labels map[string]string + ownerRefs []metav1.OwnerReference + options transfer.PodOptions + logger logr.Logger + + // TODO: this is a temporary field that needs to give away once multiple + // namespace pvcList is supported + namespace string + scc string +} + +func (tc *client) Transport() transport.Transport { + return tc.transportClient +} + +func (tc *client) PVCs() []*corev1.PersistentVolumeClaim { + pvcs := []*corev1.PersistentVolumeClaim{} + for _, pvc := range tc.pvcList.PVCs() { + pvcs = append(pvcs, pvc.Claim()) + } + return pvcs +} + +func (tc *client) Status(ctx context.Context, c ctrlclient.Client) (*transfer.Status, error) { + podList := &corev1.PodList{} + err := c.List(ctx, podList, ctrlclient.MatchingLabels(tc.labels)) + if err != nil { + return nil, err + } + + for _, pod := range podList.Items { + if len(pod.Status.ContainerStatuses) > 0 { + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.Name == "rsync" && containerStatus.State.Terminated != nil { + if containerStatus.State.Terminated.ExitCode == 0 { + return &transfer.Status{ + Completed: &transfer.Completed{ + Successful: true, + Failure: false, + FinishedAt: &containerStatus.State.Terminated.FinishedAt, + }, + }, nil + } else { + return &transfer.Status{ + Running: nil, + Completed: &transfer.Completed{ + Successful: false, + Failure: true, + FinishedAt: &containerStatus.State.Terminated.FinishedAt, + }, + }, nil + } + } + } + } + } + return nil, fmt.Errorf("unable to find the appropriate container to inspect status for rsync transfer") +} + +func (tc *client) MarkForCleanup(ctx context.Context, c ctrlclient.Client, key, value string) error { + err := tc.Transport().MarkForCleanup(ctx, c, key, value) + if err != nil { + return err + } + + err = tc.endpoint.MarkForCleanup(ctx, c, key, value) + if err != nil { + return err + } + + // update pod + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("rsync-client-%s", tc.nameSuffix), + Namespace: tc.namespace, + }, + } + err = utils.UpdateWithLabel(ctx, c, pod, key, value) + if err != nil { + return err + } + + // update service account + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, tc.nameSuffix), + Namespace: tc.namespace, + }, + } + err = utils.UpdateWithLabel(ctx, c, sa, key, value) + if err != nil { + return err + } + + // update role + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRole, tc.nameSuffix), + Namespace: tc.namespace, + }, + } + err = utils.UpdateWithLabel(ctx, c, role, key, value) + if err != nil { + return err + } + + // update rolebinding + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, tc.nameSuffix), + Namespace: tc.namespace, + }, + } + err = utils.UpdateWithLabel(ctx, c, roleBinding, key, value) + if err != nil { + return err + } + + // update secret + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncPassword, tc.nameSuffix), + Namespace: tc.namespace, + }, + } + return utils.UpdateWithLabel(ctx, c, secret, key, value) +} + +// NewClient takes PVCList, transport and endpoint object and creates all +// the resources required by the transfer client pod as well as the transfer +// pod. All the PVCs in the list will have rsync running against the server +// to sync its data. + +// The nameSuffix will be appended to the rsync client resources (pod, sa, role and rolebinding) +// hence it needs to adhere to the naming convention of kube resources. This allows for consumers +// to retry with a different suffix until retries are added to the client package + +// In order to generate the right RBAC, add the following lines to the Reconcile function annotations. +// +kubebuilder:rbac:groups=core,resources=pods;serviceaccounts;secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +func NewClient(ctx context.Context, c ctrlclient.Client, + pvcList transfer.PVCList, + t transport.Transport, + e endpoint.Endpoint, + nameSuffix string, + labels map[string]string, + ownerRefs []metav1.OwnerReference, + password string, podOptions transfer.PodOptions) (transfer.Client, error) { + tc := &client{ + username: "root", + password: password, + pvcList: pvcList, + transportClient: t, + endpoint: e, + nameSuffix: nameSuffix, + labels: labels, + ownerRefs: ownerRefs, + options: podOptions, + } + + var namespace string + namespaces := pvcList.Namespaces() + if len(namespaces) > 0 { + namespace = namespaces[0] + } + + for _, ns := range namespaces { + if ns != namespace { + return nil, fmt.Errorf("PVC list provided has pvcs in different namespaces which is not supported") + } + } + + if namespace == "" { + return nil, fmt.Errorf("ether PVC list is empty or namespace is not specified") + } + tc.namespace = namespace + + if podOptions.SCCName == nil || (podOptions.SCCName != nil && *podOptions.SCCName == "") { + //TODO: raise a warning event + } else { + tc.scc = *podOptions.SCCName + } + + tc.nameSuffix = transfer.NamespaceHashForNames(pvcList)[namespace][:10] + reconcilers := []reconcileFunc{ + tc.reconcileServiceAccount, + tc.reconcileRole, + tc.reconcileRoleBinding, + tc.reconcilePassword, + tc.reconcilePod, + } + + for _, reconcile := range reconcilers { + err := reconcile(ctx, c, tc.namespace) + if err != nil { + tc.logger.Error(err, "error reconciling rsyncServer") + return nil, err + } + } + + return tc, nil +} + +// TODO: add retries +func (tc *client) reconcilePod(ctx context.Context, c ctrlclient.Client, ns string) error { + var errs []error + rsyncOptions, err := rsyncCommandWithDefaultFlags() + if err != nil { + tc.logger.Error(err, "unable to get default flags for rsync command") + return err + } + for _, pvc := range tc.pvcList.InNamespace(ns).PVCs() { + // create Rsync command for PVC + rsyncContainerCommand := tc.getCommand(rsyncOptions, pvc) + // create rsync container + containers := []corev1.Container{ + { + Name: RsyncContainer, + Image: rsyncImage, + Command: rsyncContainerCommand, + Env: []corev1.EnvVar{ + { + Name: rsyncPasswordKey, + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{}, + Key: rsyncPasswordKey, + Optional: pointer.Bool(true), + }, + }, + }, + }, + + VolumeMounts: []corev1.VolumeMount{ + { + Name: "mnt", + MountPath: fmt.Sprintf("/mnt/%s/%s", pvc.Claim().Namespace, pvc.LabelSafeName()), + }, + { + Name: "rsync-communication", + MountPath: rsyncCommunicationMountPath, + }, + }, + }, + } + // attach transport containers + err := customizeTransportClientContainers(tc.Transport()) + if err != nil { + tc.logger.Error(err, "unable to customize Transport client containers for rsync client pod") + return err + } + containers = append(containers, tc.Transport().Containers()...) + + volumes := []corev1.Volume{ + { + Name: "mnt", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc.Claim().Name, + }, + }, + }, + { + Name: "rsync-communication", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}, + }, + }, + } + volumes = append(volumes, tc.Transport().Volumes()...) + podSpec := corev1.PodSpec{ + Containers: containers, + Volumes: volumes, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: fmt.Sprintf("%s-%s", rsyncServiceAccount, tc.nameSuffix), + } + + applyPodOptions(&podSpec, tc.options) + + pod := corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("rsync-client-%s", tc.nameSuffix), + Namespace: pvc.Claim().Namespace, + }, + } + + _, err = ctrlutil.CreateOrUpdate(ctx, c, &pod, func() error { + pod.Labels = tc.labels + // adding pvc name in annotation to avoid constraints on labels in naming + pod.Annotations = map[string]string{"pvc": pvc.Claim().Name} + pod.OwnerReferences = tc.ownerRefs + pod.Spec = podSpec + return nil + }) + errs = append(errs, err) + } + + aggregateErr := errorsutil.NewAggregate(errs) + if aggregateErr != nil { + tc.logger.Error(aggregateErr, "errors in creating pods for pvcList, please try again") + } + + return nil +} + +func (tc *client) getCommand(rsyncOptions []string, pvc transfer.PVC) []string { + // TODO: add a stub for null transport + rsyncCommand := []string{"/usr/bin/rsync"} + rsyncCommand = append(rsyncCommand, rsyncOptions...) + rsyncCommand = append(rsyncCommand, fmt.Sprintf("/mnt/%s/%s", pvc.Claim().Namespace, pvc.LabelSafeName())) + rsyncCommand = append(rsyncCommand, + fmt.Sprintf("rsync://%s@%s/%s --port %d", + tc.username, + tc.Transport().Hostname(), + pvc.LabelSafeName(), tc.Transport().ListenPort())) + rsyncCommandBashScript := fmt.Sprintf( + "trap \"touch %s/rsync-client-container-done\" EXIT SIGINT SIGTERM; timeout=120; SECONDS=0; while [ $SECONDS -lt $timeout ]; do nc -z localhost %d; rc=$?; if [ $rc -eq 0 ]; then %s; rc=$?; break; fi; done; exit $rc;", + rsyncCommunicationMountPath, + tc.Transport().ListenPort(), + strings.Join(rsyncCommand, " ")) + rsyncContainerCommand := []string{ + "/bin/bash", + "-c", + rsyncCommandBashScript, + } + return rsyncContainerCommand +} + +// customizeTransportClientContainers customizes transport's client containers for specific rsync communication +func customizeTransportClientContainers(transportClient transport.Transport) error { + switch transportClient.Type() { + case stunnel.TransportTypeStunnel: + var stunnelContainer *corev1.Container + for i := range transportClient.Containers() { + c := &transportClient.Containers()[i] + if c.Name == stunnel.Container { + stunnelContainer = c + } + } + if stunnelContainer == nil { + return fmt.Errorf("couldnt find container named %s in rsync client pod", stunnel.Container) + } + stunnelContainer.Command = []string{ + "/bin/bash", + "-c", + fmt.Sprintf(`/bin/stunnel /etc/stunnel/stunnel.conf +while true +do test -f %s/rsync-client-container-done +if [ $? -eq 0 ] +then +break +fi +done +exit 0`, rsyncCommunicationMountPath), + } + stunnelContainer.VolumeMounts = append( + stunnelContainer.VolumeMounts, + corev1.VolumeMount{ + Name: "rsync-communication", + MountPath: rsyncCommunicationMountPath, + }) + } + return nil +} + +func (tc *client) reconcileServiceAccount(ctx context.Context, c ctrlclient.Client, namespace string) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, tc.nameSuffix), + Namespace: namespace, + }, + } + _, err := ctrlutil.CreateOrUpdate(ctx, c, sa, func() error { + sa.Labels = tc.labels + sa.OwnerReferences = tc.ownerRefs + return nil + }) + return err +} + +func (tc *client) reconcileRole(ctx context.Context, c ctrlclient.Client, namespace string) error { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRole, tc.nameSuffix), + Namespace: namespace, + }, + } + _, err := ctrlutil.CreateOrUpdate(ctx, c, role, func() error { + role.OwnerReferences = tc.ownerRefs + role.Labels = tc.labels + role.Rules = []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + // Must match the name of the SCC that is deployed w/ the operator + // config/openshift/mover_scc.yaml + ResourceNames: []string{tc.scc}, + Verbs: []string{"use"}, + }, + } + return nil + }) + return err +} + +func (tc *client) reconcileRoleBinding(ctx context.Context, c ctrlclient.Client, namespace string) error { + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, tc.nameSuffix), + Namespace: namespace, + }, + } + _, err := ctrlutil.CreateOrUpdate(ctx, c, roleBinding, func() error { + roleBinding.OwnerReferences = tc.ownerRefs + roleBinding.Labels = tc.labels + roleBinding.RoleRef = rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: fmt.Sprintf("%s-%s", rsyncRole, tc.nameSuffix), + } + roleBinding.Subjects = []rbacv1.Subject{ + {Kind: "ServiceAccount", Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, tc.nameSuffix)}, + } + return nil + }) + return err +} + +func (tc *client) reconcilePassword(ctx context.Context, c ctrlclient.Client, namespace string) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncPassword, tc.nameSuffix), + Namespace: namespace, + }, + } + _, err := ctrlutil.CreateOrUpdate(ctx, c, secret, func() error { + secret.OwnerReferences = tc.ownerRefs + secret.Labels = tc.labels + secret.Data = map[string][]byte{ + rsyncPasswordKey: []byte(tc.password), + } + return nil + }) + return err +} diff --git a/transfer/rsync/client_test.go b/transfer/rsync/client_test.go new file mode 100644 index 0000000..cdd513a --- /dev/null +++ b/transfer/rsync/client_test.go @@ -0,0 +1,565 @@ +package rsync + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/backube/pvc-transfer/transfer" + "github.com/backube/pvc-transfer/transport" + "github.com/backube/pvc-transfer/transport/stunnel" + logrtesting "github.com/go-logr/logr/testing" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type fakeTransportClient struct { + transportType transport.Type +} + +func (f *fakeTransportClient) NamespacedName() types.NamespacedName { + panic("implement me") +} + +func (f *fakeTransportClient) ListenPort() int32 { + return 8080 +} + +func (f *fakeTransportClient) ConnectPort() int32 { + panic("implement me") +} + +func (f *fakeTransportClient) Containers() []corev1.Container { + return []corev1.Container{{Name: stunnel.Container}} +} + +func (f *fakeTransportClient) Volumes() []corev1.Volume { + return []corev1.Volume{{ + Name: "fakeVolume", + }} +} + +func (f *fakeTransportClient) Type() transport.Type { + return f.transportType +} + +func (f *fakeTransportClient) Credentials() types.NamespacedName { + panic("implement me") +} + +func (f *fakeTransportClient) Hostname() string { + return "foo.bar.dev" +} + +func (f *fakeTransportClient) MarkForCleanup(ctx context.Context, c ctrlclient.Client, key, value string) error { + panic("implement me") +} + +func TestClient_reconcileServiceAccount(t *testing.T) { + tests := []struct { + name string + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + wantErr bool + nameSuffix string + objects []ctrlclient.Object + }{ + { + name: "test with missing service account", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{}, + }, + { + name: "test with invalid service account", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, "foo"), + Namespace: "foo", + Labels: map[string]string{"foo": "bar"}, + OwnerReferences: []metav1.OwnerReference{}, + }, + }, + }, + }, + { + name: "test with valid service account", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, "foo"), + Namespace: "foo", + Labels: map[string]string{"test": "me"}, + OwnerReferences: testOwnerReferences(), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fakeClientWithObjects(tt.objects...) + ctx := context.WithValue(context.Background(), "test", tt.name) + + s := &client{ + logger: logrtesting.TestLogger{t}, + nameSuffix: tt.nameSuffix, + labels: tt.labels, + ownerRefs: tt.ownerRefs, + } + if err := s.reconcileServiceAccount(ctx, fakeClient, tt.namespace); (err != nil) != tt.wantErr { + t.Errorf("reconcileServiceAccount() error = %v, wantErr %v", err, tt.wantErr) + } + + sa := &corev1.ServiceAccount{} + err := fakeClient.Get(context.Background(), types.NamespacedName{ + Namespace: tt.namespace, + Name: rsyncServiceAccount + "-" + tt.nameSuffix, + }, sa) + if err != nil { + panic(fmt.Errorf("%#v should not be getting error from fake client", err)) + } + + if !reflect.DeepEqual(sa.Labels, tt.labels) { + t.Error("sa does not have the right labels") + } + if !reflect.DeepEqual(sa.OwnerReferences, tt.ownerRefs) { + t.Error("sa does not have the right owner references") + } + }) + } +} + +func TestClient_reconcileRole(t *testing.T) { + tests := []struct { + name string + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + wantErr bool + nameSuffix string + objects []ctrlclient.Object + }{ + { + name: "test with missing role", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{}, + }, + { + name: "test with invalid role", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRole, "foo"), + Namespace: "foo", + Labels: map[string]string{"foo": "bar"}, + OwnerReferences: []metav1.OwnerReference{}, + }, + }, + }, + }, + { + name: "test with valid role", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRole, "foo"), + Namespace: "foo", + Labels: map[string]string{"test": "me"}, + OwnerReferences: testOwnerReferences(), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &client{ + logger: logrtesting.TestLogger{t}, + nameSuffix: tt.nameSuffix, + labels: tt.labels, + ownerRefs: tt.ownerRefs, + } + + fakeClient := fakeClientWithObjects(tt.objects...) + ctx := context.WithValue(context.Background(), "test", tt.name) + + if err := s.reconcileRole(ctx, fakeClient, tt.namespace); (err != nil) != tt.wantErr { + t.Errorf("reconcileRole() error = %v, wantErr %v", err, tt.wantErr) + } + + role := &rbacv1.Role{} + err := fakeClient.Get(context.Background(), types.NamespacedName{ + Namespace: tt.namespace, + Name: rsyncRole + "-" + tt.nameSuffix, + }, role) + if err != nil { + panic(fmt.Errorf("%#v should not be getting error from fake client", err)) + } + + if !reflect.DeepEqual(role.Labels, tt.labels) { + t.Error("role does not have the right labels") + } + if !reflect.DeepEqual(role.OwnerReferences, tt.ownerRefs) { + t.Error("role does not have the right owner references") + } + + }) + } +} + +func Test_client_reconcileRoleBinding(t *testing.T) { + tests := []struct { + name string + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + wantErr bool + nameSuffix string + objects []ctrlclient.Object + }{ + { + name: "test with missing rolebinding", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{}, + }, + { + name: "test with invalid rolebinding", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, "foo"), + Namespace: "foo", + Labels: map[string]string{"foo": "bar"}, + OwnerReferences: []metav1.OwnerReference{}, + }, + }, + }, + }, + { + name: "test with valid rolebinding", + namespace: "foo", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, "foo"), + Namespace: "foo", + Labels: map[string]string{"test": "me"}, + OwnerReferences: testOwnerReferences(), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &client{ + logger: logrtesting.TestLogger{t}, + nameSuffix: tt.nameSuffix, + labels: tt.labels, + ownerRefs: tt.ownerRefs, + } + + fakeClient := fakeClientWithObjects(tt.objects...) + ctx := context.WithValue(context.Background(), "test", tt.name) + + if err := s.reconcileRoleBinding(ctx, fakeClient, tt.namespace); (err != nil) != tt.wantErr { + t.Errorf("reconcileRoleBinding() error = %v, wantErr %v", err, tt.wantErr) + } + rolebinding := &rbacv1.RoleBinding{} + err := fakeClient.Get(context.Background(), types.NamespacedName{ + Namespace: tt.namespace, + Name: rsyncRoleBinding + "-" + tt.nameSuffix, + }, rolebinding) + if err != nil { + panic(fmt.Errorf("%#v should not be getting error from fake client", err)) + } + + if !reflect.DeepEqual(rolebinding.Labels, tt.labels) { + t.Error("rolebinding does not have the right labels") + } + if !reflect.DeepEqual(rolebinding.OwnerReferences, tt.ownerRefs) { + t.Error("rolebinding does not have the right owner references") + } + + }) + } +} + +func Test_client_reconcilePod(t *testing.T) { + tests := []struct { + name string + username string + pvcList transfer.PVCList + transportClient transport.Transport + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + wantErr bool + nameSuffix string + listenPort int32 + objects []ctrlclient.Object + }{ + { + name: "test with no pod", + namespace: "foo", + username: "root", + pvcList: transfer.NewSingletonPVC(&corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "foo", + }, + }), + listenPort: 8080, + transportClient: &fakeTransportClient{transportType: stunnel.TransportTypeStunnel}, + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{}, + }, + { + name: "test with invalid pod", + namespace: "foo", + username: "root", + pvcList: transfer.NewSingletonPVC(&corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "foo", + }, + }), + listenPort: 8080, + transportClient: &fakeTransportClient{transportType: stunnel.TransportTypeStunnel}, + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rsync-client-data-0", + Namespace: "foo", + Labels: map[string]string{"foo": "bar"}, + OwnerReferences: []metav1.OwnerReference{}, + }, + }, + }, + }, + { + name: "test with valid pod", + namespace: "foo", + username: "root", + pvcList: transfer.NewSingletonPVC(&corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "foo", + }, + }), + listenPort: 8080, + transportClient: &fakeTransportClient{transportType: stunnel.TransportTypeStunnel}, + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rsync-client-foo", + Namespace: "foo", + Annotations: map[string]string{"pvc": "test-pvc"}, + Labels: map[string]string{"test": "me"}, + OwnerReferences: testOwnerReferences(), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fakeClientWithObjects(tt.objects...) + ctx := context.WithValue(context.Background(), "test", tt.name) + s := &client{ + logger: logrtesting.TestLogger{t}, + username: tt.username, + pvcList: tt.pvcList, + nameSuffix: tt.nameSuffix, + labels: tt.labels, + ownerRefs: tt.ownerRefs, + transportClient: tt.transportClient, + } + if err := s.reconcilePod(ctx, fakeClient, tt.namespace); (err != nil) != tt.wantErr { + t.Errorf("reconcilePod() error = %v, wantErr %v", err, tt.wantErr) + } + + pod := &corev1.Pod{} + err := fakeClient.Get(context.Background(), types.NamespacedName{ + Namespace: tt.namespace, + Name: "rsync-client-foo", + }, pod) + if err != nil { + panic(fmt.Errorf("%#v should not be getting error from fake client", err)) + } + + if !reflect.DeepEqual(pod.Labels, tt.labels) { + t.Error("pod does not have the right labels") + } + if !reflect.DeepEqual(pod.OwnerReferences, tt.ownerRefs) { + t.Error("pod does not have the right owner references") + } + if !reflect.DeepEqual(pod.Annotations, map[string]string{"pvc": tt.pvcList.PVCs()[0].Claim().Name}) { + t.Error("pod does not have the right annotations") + } + }) + } +} + +func Test_client_reconcileSecret(t *testing.T) { + tests := []struct { + name string + password string + labels map[string]string + ownerRefs []metav1.OwnerReference + namespace string + wantErr bool + nameSuffix string + objects []ctrlclient.Object + }{ + { + name: "test with missing secret", + namespace: "foo", + password: "testme123", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{}, + }, + { + name: "test with invalid secret data", + namespace: "foo", + password: "testme123", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncPassword, "foo"), + Namespace: "foo", + Labels: map[string]string{"foo": "bar"}, + OwnerReferences: []metav1.OwnerReference{}, + }, + Data: map[string][]byte{ + rsyncPasswordKey: []byte("badPassword"), + }, + }, + }, + }, + { + name: "test with valid secret", + namespace: "foo", + password: "testme123", + labels: map[string]string{"test": "me"}, + ownerRefs: testOwnerReferences(), + wantErr: false, + nameSuffix: "foo", + objects: []ctrlclient.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", rsyncPassword, "foo"), + Namespace: "foo", + Labels: map[string]string{"foo": "bar"}, + OwnerReferences: []metav1.OwnerReference{}, + }, + Data: map[string][]byte{ + rsyncPasswordKey: []byte("testme123"), + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &client{ + logger: logrtesting.TestLogger{t}, + nameSuffix: tt.nameSuffix, + labels: tt.labels, + ownerRefs: tt.ownerRefs, + password: tt.password, + } + + fakeClient := fakeClientWithObjects(tt.objects...) + ctx := context.WithValue(context.Background(), "test", tt.name) + + if err := s.reconcilePassword(ctx, fakeClient, tt.namespace); (err != nil) != tt.wantErr { + t.Errorf("reconcilePassword() error = %v, wantErr %v", err, tt.wantErr) + } + secret := &corev1.Secret{} + err := fakeClient.Get(context.Background(), types.NamespacedName{ + Namespace: tt.namespace, + Name: rsyncPassword + "-" + tt.nameSuffix, + }, secret) + if err != nil { + panic(fmt.Errorf("%#v should not be getting error from fake client", err)) + } + + if !reflect.DeepEqual(secret.Labels, tt.labels) { + t.Error("secret does not have the right labels") + } + if !reflect.DeepEqual(secret.OwnerReferences, tt.ownerRefs) { + t.Error("secret does not have the right owner references") + } + + if !reflect.DeepEqual(secret.Data, map[string][]byte{rsyncPasswordKey: []byte("testme123")}) { + t.Errorf("secret does not have the right password") + } + }) + } +} diff --git a/transfer/rsync/command_options.go b/transfer/rsync/command_options.go new file mode 100644 index 0000000..50c0ec1 --- /dev/null +++ b/transfer/rsync/command_options.go @@ -0,0 +1,213 @@ +package rsync + +import ( + "fmt" + errorsutil "k8s.io/apimachinery/pkg/util/errors" + "regexp" + "strings" +) + +const ( + optRecursive = "--recursive" + optSymLinks = "--links" + optPermissions = "--perms" + optModTimes = "--times" + optDeviceFiles = "--devices" + optSpecialFiles = "--specials" + optOwner = "--owner" + optGroup = "--group" + optHardLinks = "--hard-links" + optPartial = "--partial" + optDelete = "--delete" + optBwLimit = "--bwlimit=%d" + optInfo = "--info=%s" + optHumanReadable = "--human-readable" + optLogFile = "--log-file=%s" +) + +const ( + logFileStdOut = "/dev/stdout" +) + +type Applier interface { + ApplyTo(options *CommandOptions) error +} + +// CommandOptions defines options that can be customized in the Rsync command +type CommandOptions struct { + Recursive bool + SymLinks bool + Permissions bool + ModTimes bool + DeviceFiles bool + SpecialFiles bool + Groups bool + Owners bool + HardLinks bool + Delete bool + Partial bool + BwLimit *int + HumanReadable bool + LogFile string + Info []string + Extras []string +} + +// AsRsyncCommandOptions returns validated rsync options and validation errors as two lists +func (c *CommandOptions) AsRsyncCommandOptions() ([]string, error) { + var errs []error + opts := []string{} + if c.Recursive { + opts = append(opts, optRecursive) + } + if c.SymLinks { + opts = append(opts, optSymLinks) + } + if c.Permissions { + opts = append(opts, optPermissions) + } + if c.DeviceFiles { + opts = append(opts, optDeviceFiles) + } + if c.SpecialFiles { + opts = append(opts, optSpecialFiles) + } + if c.ModTimes { + opts = append(opts, optModTimes) + } + if c.Owners { + opts = append(opts, optOwner) + } + if c.Groups { + opts = append(opts, optGroup) + } + if c.HardLinks { + opts = append(opts, optHardLinks) + } + if c.Delete { + opts = append(opts, optDelete) + } + if c.Partial { + opts = append(opts, optPartial) + } + if c.BwLimit != nil { + if *c.BwLimit > 0 { + opts = append(opts, + fmt.Sprintf(optBwLimit, *c.BwLimit)) + } else { + errs = append(errs, fmt.Errorf("rsync bwlimit value must be a positive integer")) + } + } + if c.HumanReadable { + opts = append(opts, optHumanReadable) + } + if c.LogFile != "" { + opts = append(opts, fmt.Sprintf(optLogFile, c.LogFile)) + } + if len(c.Info) > 0 { + validatedOptions, err := filterRsyncInfoOptions(c.Info) + errs = append(errs, err) + opts = append(opts, + fmt.Sprintf( + optInfo, strings.Join(validatedOptions, ","))) + } + if len(c.Extras) > 0 { + extraOpts, err := filterRsyncExtraOptions(c.Extras) + errs = append(errs, err) + opts = append(opts, extraOpts...) + } + return opts, errorsutil.NewAggregate(errs) +} + +func filterRsyncInfoOptions(options []string) (validatedOptions []string, err error) { + var errs []error + r := regexp.MustCompile(`^[A-Z]+\d?$`) + for _, opt := range options { + if r.MatchString(opt) { + validatedOptions = append(validatedOptions, strings.TrimSpace(opt)) + } else { + errs = append(errs, fmt.Errorf("invalid value %s for Rsync option --info", opt)) + } + } + return validatedOptions, errorsutil.NewAggregate(errs) +} + +func filterRsyncExtraOptions(options []string) (validatedOptions []string, err error) { + var errs []error + r := regexp.MustCompile(`^\-{1,2}([a-z0-9]+\-){0,}?[a-z0-9]+$`) + for _, opt := range options { + if r.MatchString(opt) { + validatedOptions = append(validatedOptions, opt) + } else { + errs = append(errs, fmt.Errorf("invalid Rsync option %s", opt)) + } + } + return validatedOptions, errorsutil.NewAggregate(errs) +} + +func rsyncCommandDefaultOptions() []Applier { + return []Applier{ + ArchiveFiles(true), + StandardProgress(true), + } +} + +func (c *CommandOptions) Apply(opts ...Applier) error { + errs := []error{} + for _, opt := range opts { + if err := opt.ApplyTo(c); err != nil { + errs = append(errs, err) + } + } + return errorsutil.NewAggregate(errs) +} + +func rsyncCommandWithDefaultFlags() ([]string, error) { + c := CommandOptions{} + defaultOptions := rsyncCommandDefaultOptions() + err := c.Apply(defaultOptions...) + if err != nil { + return nil, err + } + return c.AsRsyncCommandOptions() +} + +type ArchiveFiles bool + +func (a ArchiveFiles) ApplyTo(opts *CommandOptions) error { + opts.Recursive = bool(a) + opts.SymLinks = bool(a) + opts.Permissions = bool(a) + opts.ModTimes = bool(a) + opts.Groups = bool(a) + opts.Owners = bool(a) + opts.DeviceFiles = bool(a) + opts.SpecialFiles = bool(a) + return nil +} + +type PreserveOwnership bool + +func (p PreserveOwnership) ApplyTo(opts *CommandOptions) error { + opts.Owners = bool(p) + opts.Groups = bool(p) + return nil +} + +type StandardProgress bool + +func (s StandardProgress) ApplyTo(opts *CommandOptions) error { + opts.Info = []string{ + "COPY2", "DEL2", "REMOVE2", "SKIP2", "FLIST2", "PROGRESS2", "STATS2", + } + opts.HumanReadable = true + opts.LogFile = logFileStdOut + return nil +} + +type DeleteDestination bool + +func (d DeleteDestination) ApplyTo(opts *CommandOptions) error { + opts.Delete = bool(d) + return nil +} diff --git a/transfer/rsync/rsync.go b/transfer/rsync/rsync.go index bebdf3b..e44bae2 100644 --- a/transfer/rsync/rsync.go +++ b/transfer/rsync/rsync.go @@ -10,14 +10,17 @@ const ( ) const ( - rsyncImage = "quay.io/konveyor/rsync-transfer:latest" - rsyncConfig = "backube-rsync-config" - rsyncSecretPrefix = "backube-rsync" - rsyncServiceAccount = "backube-rsync-sa" - rsyncRole = "backube-rsync-role" - rsyncRoleBinding = "backube-rsync-rolebinding" - rsyncdLogDir = "rsyncd-logs" - rsyncdLogDirPath = "/var/log/rsyncd/" + rsyncImage = "quay.io/konveyor/rsync-transfer:latest" + rsyncConfig = "backube-rsync-config" + rsyncSecretPrefix = "backube-rsync" + rsyncServiceAccount = "backube-rsync-sa" + rsyncRole = "backube-rsync-role" + rsyncPassword = "backube-rsync-password" + rsyncPasswordKey = "RSYNC_PASSWORD" + rsyncCommunicationMountPath = "/usr/share/rsync" + rsyncRoleBinding = "backube-rsync-rolebinding" + rsyncdLogDir = "rsyncd-logs" + rsyncdLogDirPath = "/var/log/rsyncd/" ) // applyPodOptions take a PodSpec and PodOptions, applies diff --git a/transfer/rsync/server.go b/transfer/rsync/server.go index 00290ce..041fb36 100644 --- a/transfer/rsync/server.go +++ b/transfer/rsync/server.go @@ -4,20 +4,22 @@ import ( "bytes" "context" "fmt" - "github.com/go-logr/logr" "strconv" "text/template" "github.com/backube/pvc-transfer/endpoint" + "github.com/backube/pvc-transfer/endpoint/route" "github.com/backube/pvc-transfer/internal/utils" "github.com/backube/pvc-transfer/transfer" "github.com/backube/pvc-transfer/transport" "github.com/backube/pvc-transfer/transport/stunnel" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -37,8 +39,8 @@ func AddToScheme(scheme *runtime.Scheme) error { // APIsToWatch give a list of APIs to watch if using this package // to deploy the endpoint -func APIsToWatch() ([]client.Object, error) { - return []client.Object{ +func APIsToWatch() ([]ctrlclient.Object, error) { + return []ctrlclient.Object{ &corev1.Secret{}, &corev1.ConfigMap{}, &corev1.Pod{}, @@ -48,9 +50,6 @@ func APIsToWatch() ([]client.Object, error) { }, nil } -// DefaultSCCName is the default name of the security context constraint -const DefaultSCCName = "pvc-transfer-mover" - const ( rsyncServerConfTemplate = `syslog facility = local7 read only = no @@ -85,7 +84,7 @@ type rsyncConfigData struct { AllowLocalhostOnly bool } -type reconcileFunc func(ctx context.Context, c client.Client, namespace string) error +type reconcileFunc func(ctx context.Context, c ctrlclient.Client, namespace string) error type server struct { username string @@ -101,6 +100,11 @@ type server struct { ownerRefs []metav1.OwnerReference options transfer.PodOptions logger logr.Logger + + // TODO: this is a temporary field that needs to give away once multiple + // namespace pvcList is supported + namespace string + scc string } func (s *server) Endpoint() endpoint.Endpoint { @@ -111,17 +115,17 @@ func (s *server) Transport() transport.Transport { return s.transportServer } -func (s *server) IsHealthy(ctx context.Context, c client.Client) (bool, error) { - return transfer.IsPodHealthy(ctx, c, client.ObjectKey{Namespace: s.pvcList.Namespaces()[0], Name: fmt.Sprintf("rsync-server-%s", s.nameSuffix)}) +func (s *server) IsHealthy(ctx context.Context, c ctrlclient.Client) (bool, error) { + return transfer.IsPodHealthy(ctx, c, ctrlclient.ObjectKey{Namespace: s.pvcList.Namespaces()[0], Name: fmt.Sprintf("rsync-server-%s", s.nameSuffix)}) } -func (s *server) Completed(ctx context.Context, c client.Client) (bool, error) { - return transfer.IsPodCompleted(ctx, c, client.ObjectKey{Namespace: s.pvcList.Namespaces()[0], Name: fmt.Sprintf("rsync-server-%s", s.nameSuffix)}, "rsync") +func (s *server) Completed(ctx context.Context, c ctrlclient.Client) (bool, error) { + return transfer.IsPodCompleted(ctx, c, ctrlclient.ObjectKey{Namespace: s.pvcList.Namespaces()[0], Name: fmt.Sprintf("rsync-server-%s", s.nameSuffix)}, "rsync") } // MarkForCleanup marks the provided "obj" to be deleted at the end of the // synchronization iteration. -func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value string) error { +func (s *server) MarkForCleanup(ctx context.Context, c ctrlclient.Client, key, value string) error { // mark endpoint for deletion err := s.Endpoint().MarkForCleanup(ctx, c, key, value) if err != nil { @@ -134,12 +138,11 @@ func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value return err } - namespace := s.pvcList.Namespaces()[0] // update configmap cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncConfig, s.nameSuffix), - Namespace: namespace, + Namespace: s.namespace, }, } err = utils.UpdateWithLabel(ctx, c, cm, key, value) @@ -150,7 +153,7 @@ func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncSecretPrefix, s.nameSuffix), - Namespace: namespace, + Namespace: s.namespace, }, } err = utils.UpdateWithLabel(ctx, c, secret, key, value) @@ -162,7 +165,7 @@ func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("rsync-server-%s", s.nameSuffix), - Namespace: namespace, + Namespace: s.namespace, }, } err = utils.UpdateWithLabel(ctx, c, pod, key, value) @@ -174,7 +177,7 @@ func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, s.nameSuffix), - Namespace: namespace, + Namespace: s.namespace, }, } err = utils.UpdateWithLabel(ctx, c, sa, key, value) @@ -186,7 +189,7 @@ func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRole, s.nameSuffix), - Namespace: namespace, + Namespace: s.namespace, }, } err = utils.UpdateWithLabel(ctx, c, role, key, value) @@ -198,7 +201,7 @@ func (s *server) MarkForCleanup(ctx context.Context, c client.Client, key, value roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, s.nameSuffix), - Namespace: namespace, + Namespace: s.namespace, }, } return utils.UpdateWithLabel(ctx, c, roleBinding, key, value) @@ -216,6 +219,52 @@ func (s *server) ListenPort() int32 { return s.listenPort } +// NewServerWithStunnelRoute creates the stunnel server resources and a route before attempting +// to create the rsync server pod and its resources. This requires the callers to call stunnel.APIsToWatch() +// and route.APIsToWatch(), to get correct list of all the APIs to be watched for the reconcilers + +// In order to generate the right RBAC, add the following lines to the Reconcile function annotations. +// +kubebuilder:rbac:groups=core,resources=services;secrets;configmaps;pods;serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete +func NewServerWithStunnelRoute(ctx context.Context, c ctrlclient.Client, logger logr.Logger, + pvcList transfer.PVCList, + labels map[string]string, + ownerRefs []metav1.OwnerReference, + password string, podOptions transfer.PodOptions) (transfer.Server, error) { + + var namespace string + namespaces := pvcList.Namespaces() + if len(namespaces) > 0 { + namespace = pvcList.Namespaces()[0] + } + + for _, ns := range namespaces { + if ns != namespace { + return nil, fmt.Errorf("PVC list provided has pvcs in different namespaces which is not supported") + } + } + + if namespace == "" { + return nil, fmt.Errorf("ether PVC list is empty or namespace is not specified") + } + hm := transfer.NamespaceHashForNames(pvcList) + e, err := route.New(ctx, c, logger, types.NamespacedName{ + Namespace: namespace, + Name: hm[namespace], + }, route.EndpointTypePassthrough, labels, ownerRefs) + if err != nil { + return nil, err + } + + t, err := stunnel.NewServer(ctx, c, logger, types.NamespacedName{Namespace: namespace, Name: hm[namespace]}, e, &transport.Options{Labels: labels, Owners: ownerRefs}) + if err != nil { + return nil, err + } + + return NewServer(ctx, c, logger, pvcList, t, e, labels, ownerRefs, password, podOptions) +} + // NewServer takes PVCList, transport and endpoint object and all // the resources required by the transfer server pod as well as the transfer // pod. All the PVCs in the list can be sync'ed via the endpoint object @@ -223,7 +272,7 @@ func (s *server) ListenPort() int32 { // In order to generate the right RBAC, add the following lines to the Reconcile function annotations. // +kubebuilder:rbac:groups=core,resources=secrets;configmaps;pods;serviceaccounts,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update;patch;delete -func NewServer(ctx context.Context, c client.Client, logger logr.Logger, +func NewServer(ctx context.Context, c ctrlclient.Client, logger logr.Logger, pvcList transfer.PVCList, t transport.Transport, e endpoint.Endpoint, @@ -254,6 +303,12 @@ func NewServer(ctx context.Context, c client.Client, logger logr.Logger, namespace = pvcList.Namespaces()[0] } + if podOptions.SCCName == nil || (podOptions.SCCName != nil && *podOptions.SCCName == "") { + //TODO: raise a warning event + } else { + r.scc = *podOptions.SCCName + } + r.nameSuffix = transfer.NamespaceHashForNames(pvcList)[namespace][:10] r.logger = logger.WithValues("rsyncServer", r.nameSuffix) @@ -262,10 +317,10 @@ func NewServer(ctx context.Context, c client.Client, logger logr.Logger, return nil, fmt.Errorf("PVC list provided has pvcs in different namespaces which is not supported") } } - if namespace == "" { return nil, fmt.Errorf("ether PVC list is empty or namespace is not specified") } + r.namespace = namespace reconcilers := []reconcileFunc{ r.reconcileConfigMap, @@ -277,7 +332,7 @@ func NewServer(ctx context.Context, c client.Client, logger logr.Logger, } for _, reconcile := range reconcilers { - err := reconcile(ctx, c, namespace) + err := reconcile(ctx, c, r.namespace) if err != nil { r.logger.Error(err, "error reconciling rsyncServer") return nil, err @@ -287,7 +342,7 @@ func NewServer(ctx context.Context, c client.Client, logger logr.Logger, return r, nil } -func (s *server) reconcileConfigMap(ctx context.Context, c client.Client, namespace string) error { +func (s *server) reconcileConfigMap(ctx context.Context, c ctrlclient.Client, namespace string) error { var rsyncConf bytes.Buffer rsyncConfTemplate, err := template.New("config").Parse(rsyncServerConfTemplate) if err != nil { @@ -326,7 +381,7 @@ func (s *server) reconcileConfigMap(ctx context.Context, c client.Client, namesp return err } -func (s *server) reconcileSecret(ctx context.Context, c client.Client, namespace string) error { +func (s *server) reconcileSecret(ctx context.Context, c ctrlclient.Client, namespace string) error { if s.password == "" { e := fmt.Errorf("password is empty") s.logger.Error(e, "unable to find password for rsyncServer") @@ -352,7 +407,7 @@ func (s *server) reconcileSecret(ctx context.Context, c client.Client, namespace return err } -func (s *server) reconcilePod(ctx context.Context, c client.Client, namespace string) error { +func (s *server) reconcilePod(ctx context.Context, c ctrlclient.Client, namespace string) error { volumeMounts := []corev1.VolumeMount{} configVolumeMounts := s.getConfigVolumeMounts() pvcVolumeMounts := s.getPVCVolumeMounts(namespace) @@ -514,7 +569,7 @@ func (s *server) getConfigVolumeMounts() []corev1.VolumeMount { } } -func (s *server) reconcileServiceAccount(ctx context.Context, c client.Client, namespace string) error { +func (s *server) reconcileServiceAccount(ctx context.Context, c ctrlclient.Client, namespace string) error { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, s.nameSuffix), @@ -529,7 +584,7 @@ func (s *server) reconcileServiceAccount(ctx context.Context, c client.Client, n return err } -func (s *server) reconcileRole(ctx context.Context, c client.Client, namespace string) error { +func (s *server) reconcileRole(ctx context.Context, c ctrlclient.Client, namespace string) error { role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRole, s.nameSuffix), @@ -545,7 +600,7 @@ func (s *server) reconcileRole(ctx context.Context, c client.Client, namespace s Resources: []string{"securitycontextconstraints"}, // Must match the name of the SCC that is deployed w/ the operator // config/openshift/mover_scc.yaml - ResourceNames: []string{DefaultSCCName}, + ResourceNames: []string{s.scc}, Verbs: []string{"use"}, }, } @@ -554,7 +609,7 @@ func (s *server) reconcileRole(ctx context.Context, c client.Client, namespace s return err } -func (s *server) reconcileRoleBinding(ctx context.Context, c client.Client, namespace string) error { +func (s *server) reconcileRoleBinding(ctx context.Context, c ctrlclient.Client, namespace string) error { roleBinding := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, s.nameSuffix), diff --git a/transfer/rsync/server_test.go b/transfer/rsync/server_test.go index 2735980..d86f392 100644 --- a/transfer/rsync/server_test.go +++ b/transfer/rsync/server_test.go @@ -3,7 +3,6 @@ package rsync import ( "context" "fmt" - logrtesting "github.com/go-logr/logr/testing" "reflect" "strings" "testing" @@ -11,13 +10,13 @@ import ( "github.com/backube/pvc-transfer/transfer" "github.com/backube/pvc-transfer/transport" "github.com/backube/pvc-transfer/transport/stunnel" + logrtesting "github.com/go-logr/logr/testing" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -71,7 +70,7 @@ func (f *fakeTransportServer) Hostname() string { panic("implement me") } -func (f *fakeTransportServer) MarkForCleanup(ctx context.Context, c client.Client, key, value string) error { +func (f *fakeTransportServer) MarkForCleanup(ctx context.Context, c ctrlclient.Client, key, value string) error { panic("implement me") } @@ -92,7 +91,7 @@ func Test_server_reconcileConfigMap(t *testing.T) { namespace string wantErr bool nameSuffix string - objects []client.Object + objects []ctrlclient.Object }{ { name: "test with no configmap", @@ -108,7 +107,7 @@ func Test_server_reconcileConfigMap(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{}, + objects: []ctrlclient.Object{}, }, { name: "test with invalid configmap", @@ -124,7 +123,7 @@ func Test_server_reconcileConfigMap(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: rsyncConfig + "-foo", @@ -149,7 +148,7 @@ func Test_server_reconcileConfigMap(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: rsyncConfig + "-foo", @@ -215,7 +214,7 @@ func Test_server_reconcileSecret(t *testing.T) { namespace string wantErr bool nameSuffix string - objects []client.Object + objects []ctrlclient.Object }{ { name: "test if password is empty", @@ -225,7 +224,7 @@ func Test_server_reconcileSecret(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: true, nameSuffix: "foo", - objects: []client.Object{}, + objects: []ctrlclient.Object{}, }, { name: "secret with invalid data", @@ -235,7 +234,7 @@ func Test_server_reconcileSecret(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "backube-rsync-foo", @@ -254,7 +253,7 @@ func Test_server_reconcileSecret(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "backube-rsync-foo", @@ -284,7 +283,7 @@ func Test_server_reconcileSecret(t *testing.T) { } err := s.reconcileSecret(ctx, fakeClient, tt.namespace) if (err != nil) != tt.wantErr { - t.Errorf("reconcileSecret() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("reconcilePassword() error = %v, wantErr %v", err, tt.wantErr) } if tt.wantErr { return @@ -325,7 +324,7 @@ func Test_server_reconcileServiceAccount(t *testing.T) { namespace string wantErr bool nameSuffix string - objects []client.Object + objects []ctrlclient.Object }{ { name: "test with missing service account", @@ -334,7 +333,7 @@ func Test_server_reconcileServiceAccount(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{}, + objects: []ctrlclient.Object{}, }, { name: "test with invalid service account", @@ -343,7 +342,7 @@ func Test_server_reconcileServiceAccount(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, "foo"), @@ -361,7 +360,7 @@ func Test_server_reconcileServiceAccount(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncServiceAccount, "foo"), @@ -415,7 +414,7 @@ func Test_server_reconcileRole(t *testing.T) { namespace string wantErr bool nameSuffix string - objects []client.Object + objects []ctrlclient.Object }{ { name: "test with missing role", @@ -424,7 +423,7 @@ func Test_server_reconcileRole(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{}, + objects: []ctrlclient.Object{}, }, { name: "test with invalid role", @@ -433,7 +432,7 @@ func Test_server_reconcileRole(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRole, "foo"), @@ -451,7 +450,7 @@ func Test_server_reconcileRole(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRole, "foo"), @@ -507,7 +506,7 @@ func Test_server_reconcileRoleBinding(t *testing.T) { namespace string wantErr bool nameSuffix string - objects []client.Object + objects []ctrlclient.Object }{ { name: "test with missing rolebinding", @@ -516,7 +515,7 @@ func Test_server_reconcileRoleBinding(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{}, + objects: []ctrlclient.Object{}, }, { name: "test with invalid rolebinding", @@ -525,7 +524,7 @@ func Test_server_reconcileRoleBinding(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, "foo"), @@ -543,7 +542,7 @@ func Test_server_reconcileRoleBinding(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", rsyncRoleBinding, "foo"), @@ -602,7 +601,7 @@ func Test_server_reconcilePod(t *testing.T) { wantErr bool nameSuffix string listenPort int32 - objects []client.Object + objects []ctrlclient.Object }{ { name: "test with no pod", @@ -619,7 +618,7 @@ func Test_server_reconcilePod(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{}, + objects: []ctrlclient.Object{}, }, { name: "test with invalid pod", @@ -636,7 +635,7 @@ func Test_server_reconcilePod(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "rsync-server-foo", @@ -662,7 +661,7 @@ func Test_server_reconcilePod(t *testing.T) { ownerRefs: testOwnerReferences(), wantErr: false, nameSuffix: "foo", - objects: []client.Object{ + objects: []ctrlclient.Object{ &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "rsync-server-foo", diff --git a/transfer/transfer.go b/transfer/transfer.go index ec3e2b3..0216b6f 100644 --- a/transfer/transfer.go +++ b/transfer/transfer.go @@ -48,6 +48,8 @@ type Client interface { // PodOptions allow callers to pass custom configuration for the transfer pods type PodOptions struct { + // For openshift environment, users can pass in the SCC name + SCCName *string // PodSecurityContext determines what GID the rsync process gets // In case of shared storage SupplementalGroups is configured to get the gid // In case of block storage FSGroup is configured to get the gid diff --git a/transport/stunnel/server.go b/transport/stunnel/server.go index f5c9483..1a653da 100644 --- a/transport/stunnel/server.go +++ b/transport/stunnel/server.go @@ -68,7 +68,7 @@ type server struct { // // Before passing the client c make sure to call AddToScheme() if core types are not already registered // In order to generate the right RBAC, add the following lines to the Reconcile function annotations. -// +kubebuilder:rbac:groups=core,resources=configmaps,secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=configmaps;secrets,verbs=get;list;watch;create;update;patch;delete func NewServer(ctx context.Context, c ctrlclient.Client, logger logr.Logger, namespacedName types.NamespacedName, e endpoint.Endpoint,