From 11dbdad3c56e0dc5e35bebe95bce412da8502335 Mon Sep 17 00:00:00 2001 From: NymanRobin Date: Fri, 9 Aug 2024 09:30:36 +0300 Subject: [PATCH] draft: add namespace scoped operator mode This adds automation and docs for restricting the operator scope from cluster wide to namespace restricted. Signed-off-by: NymanRobin --- Makefile | 8 ++ config/namespace/namespace.yaml | 2 +- .../namespaced-e2e/kustomization.yaml | 47 ++++++++ .../namespaced-manager-patch.yaml | 13 +++ .../namespaced-e2e/roles-ns-annotator.yaml | 14 +++ config/overlays/namespaced/kustomization.yaml | 46 ++++++++ .../namespaced/namespaced-manager-patch.yaml | 13 +++ .../namespaced/roles-ns-annotator.yaml | 9 ++ config/render/capm3.yaml | 15 ++- .../metal3.io/baremetalhost_controller.go | 28 ++--- .../bmceventsubscription_controller.go | 4 +- controllers/metal3.io/dataimage_controller.go | 6 +- .../hostfirmwarecomponents_controller.go | 6 +- .../hostfirmwaresettings_controller.go | 8 +- .../preprovisioningimage_controller.go | 6 +- docs/namespace-scoped-operator.md | 79 ++++++++++++++ hack/ci-e2e.sh | 6 ++ .../kustomize-namespace-annotator/Dockerfile | 11 ++ .../kustomize-namespace-annotator/README.md | 28 +++++ .../kustomize-namespace-annotator/go.mod | 31 ++++++ .../kustomize-namespace-annotator/go.sum | 92 ++++++++++++++++ .../kustomize-namespace-annotator/main.go | 101 ++++++++++++++++++ main.go | 16 ++- set_ns.py | 42 ++++++++ test/e2e/basic_ops_test.go | 20 +++- test/e2e/common.go | 32 +++--- test/e2e/config/fixture.yaml | 2 + test/e2e/config/ironic.yaml | 3 +- test/e2e/external_inspection_test.go | 20 +++- test/e2e/inspection_test.go | 20 +++- test/e2e/live_iso_test.go | 19 +++- test/e2e/provisioning_and_annotation_test.go | 20 +++- test/e2e/re_inspection_test.go | 20 +++- test/e2e/upgrade_test.go | 27 +++-- tools/deploy.sh | 18 +++- 35 files changed, 748 insertions(+), 84 deletions(-) create mode 100644 config/overlays/namespaced-e2e/kustomization.yaml create mode 100644 config/overlays/namespaced-e2e/namespaced-manager-patch.yaml create mode 100644 config/overlays/namespaced-e2e/roles-ns-annotator.yaml create mode 100644 config/overlays/namespaced/kustomization.yaml create mode 100644 config/overlays/namespaced/namespaced-manager-patch.yaml create mode 100644 config/overlays/namespaced/roles-ns-annotator.yaml create mode 100644 docs/namespace-scoped-operator.md create mode 100644 hack/tools/kustomize-namespace-annotator/Dockerfile create mode 100644 hack/tools/kustomize-namespace-annotator/README.md create mode 100644 hack/tools/kustomize-namespace-annotator/go.mod create mode 100644 hack/tools/kustomize-namespace-annotator/go.sum create mode 100644 hack/tools/kustomize-namespace-annotator/main.go create mode 100755 set_ns.py diff --git a/Makefile b/Makefile index c0659d7ba8..9f92d10ad6 100644 --- a/Makefile +++ b/Makefile @@ -209,6 +209,14 @@ manifests-generate: $(CONTROLLER_GEN) manifests-kustomize: $(KUSTOMIZE) $< build config/default > config/render/capm3.yaml +.PHONY: manifests-namespaced +manifests-namespaced: manifests-generate $(KUSTOMIZE) + $(KUSTOMIZE) build config/overlays/namespaced --enable-alpha-plugins > config/render/capm3.yaml + +.PHONY: manifests-namespaced-e2e +manifests-namespaced: manifests-generate $(KUSTOMIZE) + $(KUSTOMIZE) build config/overlays/namespaced-e2e --enable-alpha-plugins > config/render/capm3.yaml + .PHONY: set-manifest-image-bmo set-manifest-image-bmo: $(KUSTOMIZE) manifests $(info Updating container image for BMO to use ${MANIFEST_IMG}:${MANIFEST_TAG}) diff --git a/config/namespace/namespace.yaml b/config/namespace/namespace.yaml index 50a4d0febb..c8110e648b 100644 --- a/config/namespace/namespace.yaml +++ b/config/namespace/namespace.yaml @@ -3,4 +3,4 @@ kind: Namespace metadata: labels: control-plane: controller-manager - name: baremetal-operator-system \ No newline at end of file + name: baremetal-operator-system diff --git a/config/overlays/namespaced-e2e/kustomization.yaml b/config/overlays/namespaced-e2e/kustomization.yaml new file mode 100644 index 0000000000..3ce74e2201 --- /dev/null +++ b/config/overlays/namespaced-e2e/kustomization.yaml @@ -0,0 +1,47 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../e2e + +# The subjects and roleRef needs to be update here otherwise we lose name-prefix +# This is of course not ideal if the name-prefix changes +patchesStrategicMerge: + - namespaced-manager-patch.yaml + - | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: baremetal-operator-manager-rolebinding + roleRef: + kind: Role + name: baremetal-operator-manager-role + subjects: + - kind: ServiceAccount + name: baremetal-operator-controller-manager + namespace: baremetal-operator-system + +patches: +- patch: | + # Add a namespace to watch + - op: replace + path: /kind + value: RoleBinding + + target: + group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + name: baremetal-operator-manager-rolebinding + +- patch: | + # Add a namespace to watch + - op: replace + path: /kind + value: Role + target: + group: rbac.authorization.k8s.io + kind: ClusterRole + name: baremetal-operator-manager-role + +transformers: +- roles-ns-annotator.yaml diff --git a/config/overlays/namespaced-e2e/namespaced-manager-patch.yaml b/config/overlays/namespaced-e2e/namespaced-manager-patch.yaml new file mode 100644 index 0000000000..048547e5c5 --- /dev/null +++ b/config/overlays/namespaced-e2e/namespaced-manager-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - env: + - name: WATCH_NAMESPACE + value: basic-ops-test,external-inspection-test,inspection-test,live-iso-ops-test,provisioning-ops-test,re-inspection-test + name: manager diff --git a/config/overlays/namespaced-e2e/roles-ns-annotator.yaml b/config/overlays/namespaced-e2e/roles-ns-annotator.yaml new file mode 100644 index 0000000000..f8616170a2 --- /dev/null +++ b/config/overlays/namespaced-e2e/roles-ns-annotator.yaml @@ -0,0 +1,14 @@ +apiVersion: transformers.example.co/v1 +kind: ValueAnnotator +metadata: + annotations: + config.kubernetes.io/function: "container:\n image: bmo/roleannotator:1.0.0\ + \ \n" + name: notImportantHere +values: +- basic-ops-test +- external-inspection-test +- inspection-test +- live-iso-ops-test +- provisioning-ops-test +- re-inspection-test diff --git a/config/overlays/namespaced/kustomization.yaml b/config/overlays/namespaced/kustomization.yaml new file mode 100644 index 0000000000..f03a1bc612 --- /dev/null +++ b/config/overlays/namespaced/kustomization.yaml @@ -0,0 +1,46 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base +# The subjects and roleRef needs to be update here otherwise we lose name-prefix +# This is of course not ideal if the name-prefix changes +patchesStrategicMerge: + - namespaced-manager-patch.yaml + - | + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: baremetal-operator-manager-rolebinding + roleRef: + kind: Role + name: baremetal-operator-manager-role + subjects: + - kind: ServiceAccount + name: baremetal-operator-controller-manager + namespace: baremetal-operator-system + +patches: +- patch: | + # Add a namespace to watch + - op: replace + path: /kind + value: RoleBinding + + target: + group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + name: baremetal-operator-manager-rolebinding + +- patch: | + # Add a namespace to watch + - op: replace + path: /kind + value: Role + target: + group: rbac.authorization.k8s.io + kind: ClusterRole + name: baremetal-operator-manager-role + +transformers: +- roles-ns-annotator.yaml diff --git a/config/overlays/namespaced/namespaced-manager-patch.yaml b/config/overlays/namespaced/namespaced-manager-patch.yaml new file mode 100644 index 0000000000..cf0f5f9706 --- /dev/null +++ b/config/overlays/namespaced/namespaced-manager-patch.yaml @@ -0,0 +1,13 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - env: + - name: WATCH_NAMESPACE + value: metal3 + name: manager diff --git a/config/overlays/namespaced/roles-ns-annotator.yaml b/config/overlays/namespaced/roles-ns-annotator.yaml new file mode 100644 index 0000000000..09da0c705f --- /dev/null +++ b/config/overlays/namespaced/roles-ns-annotator.yaml @@ -0,0 +1,9 @@ +apiVersion: transformers.example.co/v1 +kind: ValueAnnotator +metadata: + annotations: + config.kubernetes.io/function: "container:\n image: bmo/roleannotator:1.0.0 \ + \ \n" + name: notImportantHere +values: +- metal3 diff --git a/config/render/capm3.yaml b/config/render/capm3.yaml index fa76809136..c929aea74a 100644 --- a/config/render/capm3.yaml +++ b/config/render/capm3.yaml @@ -1,3 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: metal3 +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -2167,9 +2172,10 @@ rules: - delete --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole +kind: Role metadata: name: baremetal-operator-manager-role + namespace: metal3 rules: - apiGroups: - "" @@ -2412,12 +2418,13 @@ subjects: namespace: baremetal-operator-system --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding +kind: RoleBinding metadata: name: baremetal-operator-manager-rolebinding + namespace: metal3 roleRef: apiGroup: rbac.authorization.k8s.io - kind: ClusterRole + kind: Role name: baremetal-operator-manager-role subjects: - kind: ServiceAccount @@ -2525,6 +2532,8 @@ spec: command: - /baremetal-operator env: + - name: WATCH_NAMESPACE + value: metal3 - name: POD_NAME valueFrom: fieldRef: diff --git a/controllers/metal3.io/baremetalhost_controller.go b/controllers/metal3.io/baremetalhost_controller.go index 40ece1ec62..3199b31c74 100644 --- a/controllers/metal3.io/baremetalhost_controller.go +++ b/controllers/metal3.io/baremetalhost_controller.go @@ -82,24 +82,24 @@ func (info *reconcileInfo) publishEvent(reason, message string) { info.events = append(info.events, info.host.NewEvent(reason, message)) } -// +kubebuilder:rbac:groups=metal3.io,resources=baremetalhosts,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=metal3.io,resources=baremetalhosts/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=metal3.io,resources=baremetalhosts/finalizers,verbs=update -// +kubebuilder:rbac:groups=metal3.io,resources=preprovisioningimages,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=metal3.io,resources=hardwaredata,verbs=get;list;watch;create;delete;patch;update -// +kubebuilder:rbac:groups=metal3.io,resources=hardware/finalizers,verbs=update -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update;delete -// +kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=baremetalhosts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=baremetalhosts/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=baremetalhosts/finalizers,verbs=update +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=preprovisioningimages,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=hardwaredata,verbs=get;list;watch;create;delete;patch;update +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=hardware/finalizers,verbs=update +// +kubebuilder:rbac:groups="",namespace="",resources=secrets,verbs=get;list;watch;update;delete +// +kubebuilder:rbac:groups="",namespace="",resources=events,verbs=get;list;watch;create;update;patch // Allow for managing hostfirmwaresettings, firmwareschema, bmceventsubscriptions and hostfirmwarecomponents -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwaresettings,verbs=get;list;watch;create;update;patch -//+kubebuilder:rbac:groups=metal3.io,resources=firmwareschemas,verbs=get;list;watch;create;update;patch -//+kubebuilder:rbac:groups=metal3.io,resources=bmceventsubscriptions,verbs=get;list;watch;create;update;patch -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwarecomponents,verbs=get;list;watch;create;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwaresettings,verbs=get;list;watch;create;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=firmwareschemas,verbs=get;list;watch;create;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=bmceventsubscriptions,verbs=get;list;watch;create;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwarecomponents,verbs=get;list;watch;create;update;patch // Allow for updating dataimage -// +kubebuilder:rbac:groups=metal3.io,resources=dataimages,verbs=get;list;watch;create;update;patch -// +kubebuilder:rbac:groups=metal3.io,resources=dataimages/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=dataimages,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=dataimages/status,verbs=get;update;patch // Reconcile handles changes to BareMetalHost resources. func (r *BareMetalHostReconciler) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { diff --git a/controllers/metal3.io/bmceventsubscription_controller.go b/controllers/metal3.io/bmceventsubscription_controller.go index 994cc56ddd..6041825bff 100644 --- a/controllers/metal3.io/bmceventsubscription_controller.go +++ b/controllers/metal3.io/bmceventsubscription_controller.go @@ -49,8 +49,8 @@ type BMCEventSubscriptionReconciler struct { APIReader client.Reader } -//+kubebuilder:rbac:groups=metal3.io,resources=bmceventsubscriptions,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal3.io,resources=bmceventsubscriptions/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=bmceventsubscriptions,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=bmceventsubscriptions/status,verbs=get;update;patch func (r *BMCEventSubscriptionReconciler) Reconcile(ctx context.Context, request ctrl.Request) (result ctrl.Result, err error) { reqLogger := r.Log.WithValues("bmceventsubscription", request.NamespacedName) diff --git a/controllers/metal3.io/dataimage_controller.go b/controllers/metal3.io/dataimage_controller.go index b24a8e9178..66203b2634 100644 --- a/controllers/metal3.io/dataimage_controller.go +++ b/controllers/metal3.io/dataimage_controller.go @@ -86,9 +86,9 @@ func (info *rdiInfo) publishEvent(reason, message string) { info.events = append(info.events, dataImageEvent) } -//+kubebuilder:rbac:groups=metal3.io,resources=dataimages,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal3.io,resources=dataimages/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=metal3.io,resources=dataimages/finalizers,verbs=update +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=dataimages,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=dataimages/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=dataimages/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/controllers/metal3.io/hostfirmwarecomponents_controller.go b/controllers/metal3.io/hostfirmwarecomponents_controller.go index f045e705bd..df072b9335 100644 --- a/controllers/metal3.io/hostfirmwarecomponents_controller.go +++ b/controllers/metal3.io/hostfirmwarecomponents_controller.go @@ -88,9 +88,9 @@ func (info *rhfcInfo) publishEvent(reason, message string) { info.events = append(info.events, hfcEvent) } -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwarecomponents,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwarecomponents/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwarecomponents/finalizers,verbs=update +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwarecomponents,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwarecomponents/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwarecomponents/finalizers,verbs=update // Reconcile handles changes to HostFirmwareComponents resources. func (r *HostFirmwareComponentsReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) { diff --git a/controllers/metal3.io/hostfirmwaresettings_controller.go b/controllers/metal3.io/hostfirmwaresettings_controller.go index 41ef319f8d..ab31858168 100644 --- a/controllers/metal3.io/hostfirmwaresettings_controller.go +++ b/controllers/metal3.io/hostfirmwaresettings_controller.go @@ -100,10 +100,10 @@ func (info *rInfo) publishEvent(reason, message string) { info.events = append(info.events, hfsEvent) } -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwaresettings,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal3.io,resources=hostfirmwaresettings/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=metal3.io,resources=firmwareschemas,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=metal3.io,resources=firmwareschemas/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwaresettings,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=hostfirmwaresettings/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=firmwareschemas,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=metal3.io,namespace="",resources=firmwareschemas/status,verbs=get;update;patch // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/controllers/metal3.io/preprovisioningimage_controller.go b/controllers/metal3.io/preprovisioningimage_controller.go index 1dc3cfd055..db00fe2be3 100644 --- a/controllers/metal3.io/preprovisioningimage_controller.go +++ b/controllers/metal3.io/preprovisioningimage_controller.go @@ -61,9 +61,9 @@ const ( reasonImageBuildInvalid imageConditionReason = "ImageBuildInvalid" ) -// +kubebuilder:rbac:groups=metal3.io,resources=preprovisioningimages,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=metal3.io,resources=preprovisioningimages/status,verbs=get;update;patch -// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;update +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=preprovisioningimages,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=metal3.io,namespace="",resources=preprovisioningimages/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",namespace="",resources=secrets,verbs=get;list;watch;update func (r *PreprovisioningImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := r.Log.WithValues("preprovisioningimage", req.NamespacedName) diff --git a/docs/namespace-scoped-operator.md b/docs/namespace-scoped-operator.md new file mode 100644 index 0000000000..117e262bfd --- /dev/null +++ b/docs/namespace-scoped-operator.md @@ -0,0 +1,79 @@ +# How to restrict the Baremetal Operator scope + +The guide is based on the instructions in the +[operator framework documentation][operator-scope]. + +[operator-scope]: https://sdk.operatorframework.io/docs/building-operators/golang/operator-scope/ + +## To generate manifests for namespace scoped BMO + +To generate namespace-scoped manifests, run the `manifests-namespaced` +make target. This command will create manifests specifically configured for the +namespaces defined in. It uses the `config/overlays/namespaced` kustomize +overlay to replace cluster-scoped resources with their namespace-scoped counterparts. + +## Watching resources in specific Namespaces + +When setting up the manager, you can use the environment variable +`WATCH_NAMESPACE` to restrict the operator to a specific namespace. If +`WATCH_NAMESPACE` is unset or set to an empty string, the operator will +monitor all namespaces. To limit it to a specific namespace, set +`WATCH_NAMESPACE` to that namespace. + +For example, to configure the operator to watch the same namespace where +it is deployed, update the `config/base/manager.yaml` file. Add the +following configuration under `spec.template.spec.containers.env`: + +```yaml +- name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +``` + +## Restricting Roles and permissions + +When BMO is restricted to a single namespace, the RBAC permissions need +to be updated accordingly. Instead of using `ClusterRole`, you will use +`Role`. + +The `Role` is defined in the file `config/base/rbac/role_ns.yaml`. This +file is auto-generated based on Kubebuilder RBAC markers, specifically those +in `_controller.go`. The default namespace marking is set to `""`, +which results in a `ClusterRole`. To restrict it to a specific namespace, +update this value accordingly. + +After updating the markers, generate the new manifests by running: + +```bash + make manifests +``` + +Ensure that `config/base/rbac/role_ns.yaml` has been updated to a `Role`. + +Due to limitations in Kubebuilder generation, the `RoleBinding` will not +be updated automatically.You can manually update `config/base/rbac/role_binding.yaml` +to achieve the desired outcome. Below is an example of how to modify the +`role_binding.yaml` file: + +```yaml: +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: manager-rolebinding + namespace: your-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: manager-role + namespace: your-namespace +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system +``` + +Replace `your-namespace` and other fields as necessary to match your +specific configuration. + +After this you can run `make manifests-kustomize` to get correct RoleBinding generated diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 795a6d9988..0cf8cf193e 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -58,6 +58,12 @@ export PATH="/usr/local/go/bin:${PATH}" # Build the container image with e2e tag (used in tests) IMG=quay.io/metal3-io/baremetal-operator:e2e make docker +# TODO: Put this behind a flag most likely +docker build -t bmo/roleannotator:1.0.0 \ + -f "${REPO_ROOT}/hack/tools/kustomize-namespace-annotator/Dockerfile" \ + "${REPO_ROOT}/hack/tools/kustomize-namespace-annotator/" +make manifests-namespaced-e2e + # Set up minikube minikube start --driver=kvm2 diff --git a/hack/tools/kustomize-namespace-annotator/Dockerfile b/hack/tools/kustomize-namespace-annotator/Dockerfile new file mode 100644 index 0000000000..85047c9734 --- /dev/null +++ b/hack/tools/kustomize-namespace-annotator/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.22-alpine as builder +ENV CGO_ENABLED=0 +WORKDIR /go/src/ +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN go build -ldflags '-w -s' -v -o /usr/local/bin/annotate ./ + +FROM alpine:latest +COPY --from=builder /usr/local/bin/annotate /usr/local/bin/annotate +ENTRYPOINT ["annotate"] diff --git a/hack/tools/kustomize-namespace-annotator/README.md b/hack/tools/kustomize-namespace-annotator/README.md new file mode 100644 index 0000000000..43107fb715 --- /dev/null +++ b/hack/tools/kustomize-namespace-annotator/README.md @@ -0,0 +1,28 @@ +# Kustomize Namespace Annotator for Roles and Rolebindings + +This tool provides a container that dynamically assigns namespaces to +Kubernetes roles. It is particularly useful when building a namespace-scoped +operator. + +## Usage Instructions + +Before running the `make manifests-generate-namespaced` command, you need to +build the Docker container. The container will be executed during the +generation process to modify roles. Note that this step always occurs after +any patches applied in the overlay. + +You can identify the tag used for this process from the annotator definition +in `config/overlays/namespaced/roles-ns-annotator.yaml`. + +To build the Docker container, run: + +```bash +docker build -t bmo/roleannotator:1.0.0 . +``` + +Once the container is built, you can execute the command for the namespaced +operator: + +```bash +make manifests-namespaced +``` diff --git a/hack/tools/kustomize-namespace-annotator/go.mod b/hack/tools/kustomize-namespace-annotator/go.mod new file mode 100644 index 0000000000..332aa3a35f --- /dev/null +++ b/hack/tools/kustomize-namespace-annotator/go.mod @@ -0,0 +1,31 @@ +module example.com/m + +go 1.22 + +toolchain go1.22.5 + +require sigs.k8s.io/kustomize/kyaml v0.17.2 + +require ( + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/go-errors/errors v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + golang.org/x/sys v0.18.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/hack/tools/kustomize-namespace-annotator/go.sum b/hack/tools/kustomize-namespace-annotator/go.sum new file mode 100644 index 0000000000..a612b4e9cd --- /dev/null +++ b/hack/tools/kustomize-namespace-annotator/go.sum @@ -0,0 +1,92 @@ +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/kustomize/kyaml v0.17.2 h1:+AzvoJUY0kq4QAhH/ydPHHMRLijtUKiyVyh7fOSshr0= +sigs.k8s.io/kustomize/kyaml v0.17.2/go.mod h1:9V0mCjIEYjlXuCdYsSXvyoy2BTsLESH7TlGV81S282U= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/hack/tools/kustomize-namespace-annotator/main.go b/hack/tools/kustomize-namespace-annotator/main.go new file mode 100644 index 0000000000..8f44dad3d4 --- /dev/null +++ b/hack/tools/kustomize-namespace-annotator/main.go @@ -0,0 +1,101 @@ +// $GOPATH/src/kustomize-plugin-demo/main.go +package main + +import ( + "os" + + "sigs.k8s.io/kustomize/kyaml/fn/framework" + "sigs.k8s.io/kustomize/kyaml/fn/framework/command" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type ValueAnnotator struct { + Values []string `yaml:"values" json:"values"` +} + +func main() { + config := new(ValueAnnotator) + fn := func(items []*yaml.RNode) ([]*yaml.RNode, error) { + var removeIndices []int + for i := range items { + + if items[i].GetKind() == "Role" && items[i].GetName() == "baremetal-operator-manager-role" { + updatedRoles, err := duplicateResourceOverNamespaces(items[i], config) + removeIndices = append(removeIndices, i) + items = append(items, updatedRoles...) + if err != nil { + return nil, err + } + } + + if items[i].GetKind() == "RoleBinding" && items[i].GetName() == "baremetal-operator-manager-rolebinding" { + updatedRoles, err := duplicateResourceOverNamespaces(items[i], config) + removeIndices = append(removeIndices, i) + items = append(items, updatedRoles...) + if err != nil { + return nil, err + } + } + + items[i].GetDataMap() + + } + + // Remove the originally generated roles to not have duplicates, alternativley these could also be modified + for i := len(removeIndices); i > 0; i-- { + if len(removeIndices) > 0 { + indexToRemove := removeIndices[i-1] + items = append(items[:indexToRemove], items[indexToRemove+1:]...) + } + } + + // Ensure all namespaces are defined, as roles cannot be applied without existing namespaces + for _, namespace := range config.Values { + namespaceYaml := createNamespaceDefinition(namespace) + items = append(items, namespaceYaml) + } + + return items, nil + } + p := framework.SimpleProcessor{Config: config, Filter: kio.FilterFunc(fn)} + cmd := command.Build(p, command.StandaloneDisabled, false) + command.AddGenerateDockerfile(cmd) + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} + +func duplicateResourceOverNamespaces(roleConfig *yaml.RNode, config *ValueAnnotator) ([]*yaml.RNode, error) { + var new_roles []*yaml.RNode + for _, value := range config.Values { + testNode := roleConfig.Copy() + err := testNode.PipeE(yaml.SetK8sNamespace(value)) + if err != nil { + return nil, err + } + new_roles = append(new_roles, testNode) + } + return new_roles, nil +} + +func createNamespaceDefinition(namespace string) *yaml.RNode { + namespaceNode := yaml.NewRNode(&yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "apiVersion"}, + {Kind: yaml.ScalarNode, Value: "v1"}, + {Kind: yaml.ScalarNode, Value: "kind"}, + {Kind: yaml.ScalarNode, Value: "Namespace"}, + {Kind: yaml.ScalarNode, Value: "metadata"}, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "name"}, + {Kind: yaml.ScalarNode, Value: namespace}, + }, + }, + }, + }) + return namespaceNode +} diff --git a/main.go b/main.go index bc32a7e5f8..cdef21aacb 100644 --- a/main.go +++ b/main.go @@ -205,11 +205,19 @@ func main() { restConfig.QPS = float32(restConfigQPS) restConfig.Burst = restConfigBurst - var watchNamespaces map[string]cache.Config - if watchNamespace != "" { - watchNamespaces = map[string]cache.Config{ - watchNamespace: {}, + watchNamespaces := make(map[string]cache.Config) + + if strings.Contains(watchNamespace, ",") { + setupLog.Info("Manager set up with multiple namespaces to watch", "namespaces", watchNamespace) + namespaces := strings.Split(watchNamespace, ",") + for _, namespace := range namespaces { + watchNamespaces[namespace] = cache.Config{} } + } else if watchNamespace != "" { + setupLog.Info("Manager set up to watch a single namespace", "namespace", watchNamespace) + watchNamespaces[watchNamespace] = cache.Config{} + } else { + setupLog.Info("Manager set up with cluster scope") } ctrlOpts := ctrl.Options{ diff --git a/set_ns.py b/set_ns.py new file mode 100755 index 0000000000..a34d4b00c7 --- /dev/null +++ b/set_ns.py @@ -0,0 +1,42 @@ +import os +import sys + +import yaml + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +ANNOTATOR_PATH = os.path.join(CURRENT_DIR, "config", "overlays", "namespaced", "roles-ns-annotator.yaml") +MANAGER_PATCH_PATH = os.path.join(CURRENT_DIR, "config", "overlays", "namespaced", "namespaced-manager-patch.yaml") + + +def update_annotator_file(namespaces_to_set): + # Load the YAML file + with open(ANNOTATOR_PATH, 'r') as file: + doc = yaml.safe_load(file) + + doc['values'] = [ns.strip() for ns in namespaces_to_set.split(',')] + + with open(ANNOTATOR_PATH, 'w') as file: + yaml.dump(doc, file, default_flow_style=False) + +def set_watch_namespace_env_var(namespaces_to_set): + with open(MANAGER_PATCH_PATH, 'r') as file: + doc = yaml.safe_load(file) + + # Dicts are always shallow copies and our patch is limited to one + # one container and one environement value + containers_spec = doc["spec"]["template"]["spec"]["containers"] + containers_spec[0]["env"][0]["value"] = namespaces_to_set + + with open(MANAGER_PATCH_PATH, 'w') as file: + yaml.dump(doc, file, default_flow_style=False) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: {} \"namespace1,namespace2\"") + sys.exit(1) + + namespaces_to_set = sys.argv[1] + + update_annotator_file(namespaces_to_set) + set_watch_namespace_env_var(namespaces_to_set) diff --git a/test/e2e/basic_ops_test.go b/test/e2e/basic_ops_test.go index 7c484beec5..7ff56138a1 100644 --- a/test/e2e/basic_ops_test.go +++ b/test/e2e/basic_ops_test.go @@ -26,12 +26,23 @@ var _ = Describe("basic", Label("required", "basic"), func() { poweroffAnnotation = "reboot.metal3.io/poweroff" ) BeforeEach(func() { - namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, - }) + } + + if namespaced == TruthyString { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) }) It("should control power cycle of BMH though annotations", func() { @@ -108,7 +119,8 @@ var _ = Describe("basic", Label("required", "basic"), func() { AfterEach(func() { if !skipCleanup { - cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) } }) }) diff --git a/test/e2e/common.go b/test/e2e/common.go index ef275aec59..adf69019cd 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -22,14 +22,16 @@ import ( "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/kustomize/api/krusty" + kustomizeTypes "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/filesys" ) type PowerState string const ( - PoweredOn PowerState = "on" - PoweredOff PowerState = "off" + PoweredOn PowerState = "on" + PoweredOff PowerState = "off" + TruthyString string = "true" ) func isUndesiredState(currentState metal3api.ProvisioningState, undesiredStates []metal3api.ProvisioningState) bool { @@ -118,15 +120,17 @@ func WaitForNamespaceDeleted(ctx context.Context, input WaitForNamespaceDeletedI }, intervals...).Should(BeTrue()) } -func cleanup(ctx context.Context, clusterProxy framework.ClusterProxy, namespace *corev1.Namespace, cancelWatches context.CancelFunc, intervals ...interface{}) { - framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ - Deleter: clusterProxy.GetClient(), - Name: namespace.Name, - }) - WaitForNamespaceDeleted(ctx, WaitForNamespaceDeletedInput{ - Getter: clusterProxy.GetClient(), - Namespace: *namespace, - }, intervals...) +func Cleanup(ctx context.Context, clusterProxy framework.ClusterProxy, namespace *corev1.Namespace, cancelWatches context.CancelFunc, namespaced string, intervals ...interface{}) { + if namespaced != TruthyString { + framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{ + Deleter: clusterProxy.GetClient(), + Name: namespace.Name, + }) + WaitForNamespaceDeleted(ctx, WaitForNamespaceDeletedInput{ + Getter: clusterProxy.GetClient(), + Namespace: *namespace, + }, intervals...) + } cancelWatches() } @@ -146,7 +150,11 @@ func WaitForBmhInPowerState(ctx context.Context, input WaitForBmhInPowerStateInp } func BuildKustomizeManifest(source string) ([]byte, error) { - kustomizer := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + options := krusty.MakeDefaultOptions() + options.PluginConfig = kustomizeTypes.EnabledPluginConfig(kustomizeTypes.BploUseStaticallyLinked) + + kustomizer := krusty.MakeKustomizer(options) + fSys := filesys.MakeFsOnDisk() resources, err := kustomizer.Run(fSys, source) if err != nil { diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index bdf38e1aa0..bdc809d1f5 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -22,6 +22,8 @@ variables: # This setting is for the separate cluster that are used for upgrade tests UPGRADE_DEPLOY_CERT_MANAGER: "true" + NAMESPACE_SCOPED: "true" + IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" ISO_IMAGE_URL: "http://192.168.222.1/cirros.iso" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index 6c275faece..c89f95a6e9 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -20,7 +20,7 @@ variables: DEPLOY_IRONIC: "true" DEPLOY_BMO: "true" DEPLOY_CERT_MANAGER: "true" - BMO_KUSTOMIZATION: "../../config/overlays/e2e" + BMO_KUSTOMIZATION: "../../config/overlays/namespaced-e2e" IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e" # This setting is for the separate cluster that are used for upgrade tests @@ -34,6 +34,7 @@ variables: SSH_USERNAME: "root" SSH_PRIV_KEY: "./images/ssh_testkey" SSH_PUB_KEY: "./images/ssh_testkey.pub" + NAMESPACE_SCOPED: "true" intervals: inspection/wait-unmanaged: ["1m", "5s"] diff --git a/test/e2e/external_inspection_test.go b/test/e2e/external_inspection_test.go index 6b6c67fdd5..9ac589ad96 100644 --- a/test/e2e/external_inspection_test.go +++ b/test/e2e/external_inspection_test.go @@ -178,12 +178,23 @@ var _ = Describe("External Inspection", Label("required", "external-inspection") cancelWatches context.CancelFunc ) BeforeEach(func() { - namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, - }) + } + + if namespaced == TruthyString { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) }) It("should skip inspection and become available when a BMH has annotations with hardware details and inspection disabled", func() { @@ -236,7 +247,8 @@ var _ = Describe("External Inspection", Label("required", "external-inspection") AfterEach(func() { if !skipCleanup { - cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) } }) }) diff --git a/test/e2e/inspection_test.go b/test/e2e/inspection_test.go index 4a6131aa69..26d3695f5b 100644 --- a/test/e2e/inspection_test.go +++ b/test/e2e/inspection_test.go @@ -22,12 +22,23 @@ var _ = Describe("Inspection", Label("required", "inspection"), func() { cancelWatches context.CancelFunc ) BeforeEach(func() { - namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, - }) + } + + if namespaced == TruthyString { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) }) It("should put BMH without BMC credentials in unmanaged state", func() { @@ -125,7 +136,8 @@ var _ = Describe("Inspection", Label("required", "inspection"), func() { AfterEach(func() { if !skipCleanup { - cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) } }) }) diff --git a/test/e2e/live_iso_test.go b/test/e2e/live_iso_test.go index 4c07028895..b0404da8f2 100644 --- a/test/e2e/live_iso_test.go +++ b/test/e2e/live_iso_test.go @@ -40,12 +40,22 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() { imageURL = e2eConfig.GetVariable("ISO_IMAGE_URL") - namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, - }) + } + + if namespaced == TruthyString { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) }) It("should provision a BMH with live ISO and then deprovision it", func() { @@ -135,7 +145,8 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() { AfterEach(func() { if !skipCleanup { - cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) } }) }) diff --git a/test/e2e/provisioning_and_annotation_test.go b/test/e2e/provisioning_and_annotation_test.go index 2232a34e68..236329a1d2 100644 --- a/test/e2e/provisioning_and_annotation_test.go +++ b/test/e2e/provisioning_and_annotation_test.go @@ -30,12 +30,23 @@ var _ = Describe("Provision, detach, recreate from status and deprovision", Labe ) BeforeEach(func() { - namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, - }) + } + + if namespaced == TruthyString { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) }) It("provisions a BMH, applies detached and status annotations, then deprovisions", func() { @@ -255,7 +266,8 @@ var _ = Describe("Provision, detach, recreate from status and deprovision", Labe AfterEach(func() { if !skipCleanup { - cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) } }) }) diff --git a/test/e2e/re_inspection_test.go b/test/e2e/re_inspection_test.go index bec9cfd11a..0148899d09 100644 --- a/test/e2e/re_inspection_test.go +++ b/test/e2e/re_inspection_test.go @@ -27,12 +27,23 @@ var _ = Describe("Re-Inspection", Label("required", "re-inspection"), func() { wrongHostName = "wrongHostName" ) BeforeEach(func() { - namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), ClientSet: clusterProxy.GetClientSet(), - Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, - }) + } + + if namespaced == TruthyString { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) }) It("should re-inspect the annotated BMH", func() { @@ -106,7 +117,8 @@ var _ = Describe("Re-Inspection", Label("required", "re-inspection"), func() { AfterEach(func() { if !skipCleanup { - cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) } }) }) diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 7c420290d8..1e42acf77b 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -240,12 +240,22 @@ func RunUpgradeTest(ctx context.Context, input *BMOIronicUpgradeInput, upgradeCl }) } - namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ - Creator: upgradeClusterProxy.GetClient(), - ClientSet: upgradeClusterProxy.GetClientSet(), - Name: fmt.Sprintf("upgrade-%s-%s", input.UpgradeEntityName, util.RandomString(6)), - LogFolder: testCaseArtifactFolder, - }) + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + + namespaceInput := framework.CreateNamespaceAndWatchEventsInput{ + Creator: clusterProxy.GetClient(), + ClientSet: clusterProxy.GetClientSet(), + LogFolder: artifactFolder, + } + + if namespaced == "true" { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, "test") + namespaceInput.IgnoreAlreadyExists = true + } else { + namespaceInput.Name = fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + } + + namespace, cancelWatches := framework.CreateNamespaceAndWatchEvents(ctx, namespaceInput) By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ @@ -414,7 +424,10 @@ var _ = Describe("Upgrade", Label("optional", "upgrade"), func() { ) AfterEach(func() { - cleanup(ctx, upgradeClusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + if !skipCleanup { + namespaced := e2eConfig.GetVariable("NAMESPACE_SCOPED") + Cleanup(ctx, clusterProxy, namespace, cancelWatches, namespaced, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + } }) }) diff --git a/tools/deploy.sh b/tools/deploy.sh index cf763062ae..792dd262d2 100755 --- a/tools/deploy.sh +++ b/tools/deploy.sh @@ -197,8 +197,14 @@ fi if [[ "${DEPLOY_BMO}" == "true" ]]; then # Create a temporary overlay where we can make changes. pushd "${TEMP_BMO_OVERLAY}" - ${KUSTOMIZE} create --resources=../../base,../../namespace \ - --namespace=baremetal-operator-system + + # TODO: ADD FLAG FOR NAMESPACE SCOPED + # TODO: ADD ANOTHER FLAG FOR WHICH NAMESPACES + + #${KUSTOMIZE} create --resources=../../base,../../namespace \ + #--namespace=baremetal-operator-system + + ${KUSTOMIZE} create --resources=../../overlays/namespaced,../../namespace if [ "${DEPLOY_BASIC_AUTH}" == "true" ]; then ${KUSTOMIZE} edit add component ../../components/basic-auth @@ -222,8 +228,14 @@ if [[ "${DEPLOY_BMO}" == "true" ]]; then # This is to keep the current behavior of using the ironic.env file for the configmap cp "${SCRIPTDIR}/config/default/ironic.env" "${TEMP_BMO_OVERLAY}/ironic.env" ${KUSTOMIZE} edit add configmap ironic --behavior=create --from-env-file=ironic.env + + # Need to build the kustomize role annotator plugin + docker build -t bmo/roleannotator:1.0.0 \ + -f "${SCRIPTDIR}/hack/tools/kustomize-namespace-annotator/Dockerfile" \ + "${SCRIPTDIR}/hack/tools/kustomize-namespace-annotator/" + # shellcheck disable=SC2086 - ${KUSTOMIZE} build "${TEMP_BMO_OVERLAY}" | kubectl apply ${KUBECTL_ARGS} -f - + ${KUSTOMIZE} build --enable-alpha-plugins "${TEMP_BMO_OVERLAY}" | kubectl apply ${KUBECTL_ARGS} -f - popd fi