diff --git a/deploy/charts/csi-driver-spiffe/README.md b/deploy/charts/csi-driver-spiffe/README.md index e3864fb..643140e 100644 --- a/deploy/charts/csi-driver-spiffe/README.md +++ b/deploy/charts/csi-driver-spiffe/README.md @@ -94,6 +94,15 @@ Verbosity of cert-manager-csi-driver logging. > ``` Duration requested for requested certificates. +#### **app.runtimeIssuanceConfigMap** ~ `string` +> Default value: +> ```yaml +> "" +> ``` + +Name of a ConfigMap in the installation namespace to watch, providing runtime configuration of an issuer to use. + +The "issuer-name", "issuer-kind" and "issuer-group" keys must be present in the ConfigMap for it to be used. #### **app.extraCertificateRequestAnnotations** ~ `unknown` > Default value: > ```yaml diff --git a/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml b/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml index 4d97290..51fe910 100644 --- a/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml +++ b/deploy/charts/csi-driver-spiffe/templates/daemonset.yaml @@ -83,6 +83,8 @@ spec: - --node-id=$(NODE_ID) - --endpoint=$(CSI_ENDPOINT) - --data-root=csi-data-dir + - "--runtime-issuance-config-map-name={{.Values.app.runtimeIssuanceConfigMap}}" + - "--runtime-issuance-config-map-namespace={{.Release.Namespace}}" {{- if .Values.app.extraCertificateRequestAnnotations }} - --extra-certificate-request-annotations={{ .Values.app.extraCertificateRequestAnnotations }} {{- end }} diff --git a/deploy/charts/csi-driver-spiffe/templates/role.yaml b/deploy/charts/csi-driver-spiffe/templates/role.yaml index 8df5440..4c00dd3 100644 --- a/deploy/charts/csi-driver-spiffe/templates/role.yaml +++ b/deploy/charts/csi-driver-spiffe/templates/role.yaml @@ -1,3 +1,21 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cert-manager-csi-driver-spiffe.labels" . | nindent 4 }} +rules: +{{- if .Values.app.runtimeIssuanceConfigMap }} +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + resourceNames: ["{{.Values.app.runtimeIssuanceConfigMap}}"] +{{- end }} + + +--- + kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml b/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml index c5f5f0f..8e1a0d0 100644 --- a/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml +++ b/deploy/charts/csi-driver-spiffe/templates/rolebinding.yaml @@ -1,3 +1,21 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cert-manager-csi-driver-spiffe.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} +subjects: +- kind: ServiceAccount + name: {{ include "cert-manager-csi-driver-spiffe.name" . }} + namespace: {{ .Release.Namespace }} + +--- + kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/deploy/charts/csi-driver-spiffe/values.schema.json b/deploy/charts/csi-driver-spiffe/values.schema.json index 054730e..6bb3334 100644 --- a/deploy/charts/csi-driver-spiffe/values.schema.json +++ b/deploy/charts/csi-driver-spiffe/values.schema.json @@ -65,6 +65,9 @@ "name": { "$ref": "#/$defs/helm-values.app.name" }, + "runtimeIssuanceConfigMap": { + "$ref": "#/$defs/helm-values.app.runtimeIssuanceConfigMap" + }, "trustDomain": { "$ref": "#/$defs/helm-values.app.trustDomain" } @@ -447,6 +450,11 @@ "description": "The name for the CSI driver installation.", "type": "string" }, + "helm-values.app.runtimeIssuanceConfigMap": { + "default": "", + "description": "Name of a ConfigMap in the installation namespace to watch, providing runtime configuration of an issuer to use.\n\nThe \"issuer-name\", \"issuer-kind\" and \"issuer-group\" keys must be present in the ConfigMap for it to be used.", + "type": "string" + }, "helm-values.app.trustDomain": { "default": "cluster.local", "description": "The Trust Domain for this driver.", diff --git a/deploy/charts/csi-driver-spiffe/values.yaml b/deploy/charts/csi-driver-spiffe/values.yaml index 23c5c9e..f6d530e 100644 --- a/deploy/charts/csi-driver-spiffe/values.yaml +++ b/deploy/charts/csi-driver-spiffe/values.yaml @@ -48,6 +48,14 @@ app: logLevel: 1 # 1-5 # Duration requested for requested certificates. certificateRequestDuration: 1h + + # Name of a ConfigMap in the installation namespace to watch, providing + # runtime configuration of an issuer to use. + # + # The "issuer-name", "issuer-kind" and "issuer-group" keys must be present in + # the ConfigMap for it to be used. + runtimeIssuanceConfigMap: "" + # List of annotations to add to certificate requests # # For example: diff --git a/internal/csi/app/app.go b/internal/csi/app/app.go index 5de4580..a51d259 100644 --- a/internal/csi/app/app.go +++ b/internal/csi/app/app.go @@ -71,7 +71,10 @@ func NewCommand(ctx context.Context) *cobra.Command { TrustDomain: opts.CertManager.TrustDomain, CertificateRequestAnnotations: opts.CertManager.CertificateRequestAnnotations, CertificateRequestDuration: opts.CertManager.CertificateRequestDuration, - IssuerRef: opts.CertManager.IssuerRef, + IssuerRef: &opts.CertManager.IssuerRef, + + IssuanceConfigMapName: opts.CertManager.IssuanceConfigMapName, + IssuanceConfigMapNamespace: opts.CertManager.IssuanceConfigMapNamespace, CertificateFileName: opts.Volume.CertificateFileName, KeyFileName: opts.Volume.KeyFileName, diff --git a/internal/csi/app/options/options.go b/internal/csi/app/options/options.go index 1c83161..1bdfa18 100644 --- a/internal/csi/app/options/options.go +++ b/internal/csi/app/options/options.go @@ -56,6 +56,12 @@ type OptionsDriver struct { // OptionsCertManager is options specific to cert-manager CertificateRequests. type OptionsCertManager struct { + // IssuanceConfigMapName is the name of a ConfigMap to watch for configuration options. The ConfigMap is expected to be in the same namespace as the csi-driver-spiffe pod. + IssuanceConfigMapName string + + // IssuanceConfigMapNamespace is the namespace where the runtime configuration ConfigMap is located + IssuanceConfigMapNamespace string + // TrustDomain is the trust domain of this SPIFFE PKI. The TrustDomain will // appear in signed certificate's URI SANs. TrustDomain string @@ -113,6 +119,10 @@ func (o *Options) addDriverFlags(fs *pflag.FlagSet) { } func (o *Options) addCertManagerFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.CertManager.IssuanceConfigMapName, "runtime-issuance-config-map-name", "", "Name of a ConfigMap to watch at runtime for issuer details. If such a ConfigMap is found, overrides issuer-name, issuer-kind and issuer-group") + + fs.StringVar(&o.CertManager.IssuanceConfigMapNamespace, "runtime-issuance-config-map-namespace", "", "Namespace for ConfigMap to be watched at runtime for issuer details") + fs.StringVar(&o.CertManager.TrustDomain, "trust-domain", "cluster.local", "The trust domain that will be requested for on created CertificateRequests.") fs.DurationVar(&o.CertManager.CertificateRequestDuration, "certificate-request-duration", time.Hour, diff --git a/internal/csi/driver/driver.go b/internal/csi/driver/driver.go index 31ed3c0..0e4545f 100644 --- a/internal/csi/driver/driver.go +++ b/internal/csi/driver/driver.go @@ -38,8 +38,12 @@ import ( "github.com/cert-manager/csi-lib/storage" "github.com/go-logr/logr" "gopkg.in/square/go-jose.v2/jwt" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/rest" "k8s.io/utils/clock" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/cert-manager/csi-driver-spiffe/internal/annotations" "github.com/cert-manager/csi-driver-spiffe/internal/csi/rootca" @@ -73,7 +77,7 @@ type Options struct { CertificateRequestDuration time.Duration // IssuerRef is the IssuerRef used when creating CertificateRequests. - IssuerRef cmmeta.ObjectReference + IssuerRef *cmmeta.ObjectReference // CertificateFileName is the name of the file that the signed certificate // will be written to inside the Pod's volume. @@ -98,6 +102,12 @@ type Options struct { // CAFileName. If the root CA certificate data changes, all managed volume's // file will be updated. RootCAs rootca.Interface + + // IssuanceConfigMapName is the name of the ConfigMap to watch for issuance configuration. + IssuanceConfigMapName string + + // IssuanceConfigMapNamespace is the namespace of the ConfigMap to watch for issuance configuration + IssuanceConfigMapNamespace string } // Driver is used for running the actual CSI driver. Driver will respond to @@ -117,9 +127,20 @@ type Driver struct { // created CertificateRequests. certificateRequestDuration time.Duration - // issuerRef is the issuerRef that will be set on all created - // CertificateRequests. - issuerRef cmmeta.ObjectReference + // activeIssuerRef is the issuerRef that will be set on all created CertificateRequests. + // Can be changed at runtime via runtime configuration (i.e. reading from a ConfigMap) + // Not to be confused with originalIssuerRef, which is an issuerRef optionally passed in + // via CLI args. + activeIssuerRef *cmmeta.ObjectReference + + // originalIssuerRef is the issuerRef passed into the driver at startup. This will be used + // if no runtime configuration (ConfigMap configuration) is found, or if the ConfigMap for + // runtime configuration is deleted. + originalIssuerRef *cmmeta.ObjectReference + + // activeIssuerRefMutex is used to control changes to the activeIssuerRef which can happen + // concurrently with a request to issue a new cert + activeIssuerRefMutex *sync.RWMutex // certFileName, keyFileName, caFileName are the names used when writing file // to volumes. @@ -138,6 +159,17 @@ type Driver struct { // camanager is used to update all managed volumes with the current root CA // certificates PEM. camanager *camanager + + // kubernetesClient is used to watch ConfigMaps for issuance configuration + kubernetesClient client.WithWatch + + // issuanceConfigMapName is the name of a ConfigMap which will be + // watched for issuance configuration at runtime + issuanceConfigMapName string + + // issuanceConfigMapNamespace is the name of a ConfigMap which will be + // watched for issuance configuration at runtime + issuanceConfigMapNamespace string } // New constructs a new Driver instance. @@ -148,16 +180,39 @@ func New(log logr.Logger, opts Options) (*Driver, error) { // don't exit, not a fatal error as sanitizeAnnotations will trim bad annotations } + originalIssuerRef, err := handleOriginalIssuerRef(opts.IssuerRef) + if err != nil && err != errNoOriginalIssuer { + return nil, err + } + + if originalIssuerRef == nil && (opts.IssuanceConfigMapName == "" || opts.IssuanceConfigMapNamespace == "") { + // if no install-time issuer was configured, runtime issuance details are not optional + return nil, fmt.Errorf("runtime issuance configuration is required if no issuer is provided at startup") + } + d := &Driver{ log: log.WithName("csi"), trustDomain: opts.TrustDomain, certFileName: opts.CertificateFileName, keyFileName: opts.KeyFileName, - issuerRef: opts.IssuerRef, - rootCAs: opts.RootCAs, + + // we check if we can set activeIssuerRef later + activeIssuerRef: nil, + originalIssuerRef: originalIssuerRef, + + activeIssuerRefMutex: &sync.RWMutex{}, + + rootCAs: opts.RootCAs, certificateRequestDuration: opts.CertificateRequestDuration, certificateRequestAnnotations: sanitizedAnnotations, + + issuanceConfigMapName: opts.IssuanceConfigMapName, + issuanceConfigMapNamespace: opts.IssuanceConfigMapNamespace, + } + + if d.originalIssuerRef != nil { + d.activeIssuerRef = d.originalIssuerRef } if len(d.certFileName) == 0 { @@ -194,6 +249,13 @@ func New(log logr.Logger, opts Options) (*Driver, error) { return nil, fmt.Errorf("failed to build cert-manager client: %w", err) } + k8sClient, err := client.NewWithWatch(opts.RestConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to build kubernetes watcher client: %w", err) + } + + d.kubernetesClient = k8sClient + mngrLog := d.log.WithName("manager") d.driver, err = driver.New(opts.Endpoint, d.log.WithName("driver"), driver.Options{ DriverName: opts.DriverName, @@ -222,7 +284,150 @@ func New(log logr.Logger, opts Options) (*Driver, error) { return d, nil } -// Run is a blocking func that run the CSI driver. +// watchRuntimeConfigurationSource should be called in a goroutine to watch a ConfigMap for runtime configuration +func (d *Driver) watchRuntimeConfigurationSource(ctx context.Context) { + logger := d.log.WithName("runtime-config-watcher").WithValues("config-map-name", d.issuanceConfigMapName, "config-map-namespace", d.issuanceConfigMapNamespace) + +LOOP: + for { + logger.Info("Starting / restarting watcher for runtime configuration") + cmList := &corev1.ConfigMapList{} + + // First create a watcher. This is in a labelled loop in case the watcher dies for some reason + // while we're running - in that case, we don't want to give up entirely on watching for runtime config + // but instead we want to recreate the watcher. + + watcher, err := d.kubernetesClient.Watch(ctx, cmList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", d.issuanceConfigMapName), + Namespace: d.issuanceConfigMapNamespace, + }) + + if err != nil { + logger.Error(err, "Failed to create ConfigMap watcher; will retry in 5s") + time.Sleep(5 * time.Second) + continue + } + + for { + // Now loop indefinitely until the main context cancels or we get an event to process. + // If the main context cancels, we break out of the outer loop and this function returns. + // If we get an event, we first check whether the channel closed. If so, we recreate the watcher by continuing + // the outer loop. + select { + case <-ctx.Done(): + logger.Info("Received context cancellation, shutting down runtime configuration watcher") + watcher.Stop() + break LOOP + + case event, open := <-watcher.ResultChan(): + if !open { + logger.Info("Received closed channel from ConfigMap watcher, will recreate") + watcher.Stop() + continue LOOP + } + + switch event.Type { + case watch.Deleted: + d.handleRuntimeConfigIssuerDeletion(logger) + + case watch.Added: + err := d.handleRuntimeConfigIssuerChange(logger, event) + if err != nil { + logger.Error(err, "Failed to handle new runtime configuration for issuerRef") + } + + case watch.Modified: + err := d.handleRuntimeConfigIssuerChange(logger, event) + if err != nil { + logger.Error(err, "Failed to handle runtime configuration issuerRef change") + } + + case watch.Bookmark: + // Ignore + + case watch.Error: + err, ok := event.Object.(error) + if !ok { + logger.Error(nil, "Got an error event when watching runtime configuration but unable to determine further information") + } else { + logger.Error(err, "Got an error event when watching runtime configuration") + } + + default: + logger.Info("Got unknown event for runtime configuration ConfigMap; ignoring", "event-type", string(event.Type)) + } + } + } + } + + logger.Info("Stopped runtime configuration watcher") +} + +const ( + issuerNameKey = "issuer-name" + issuerKindKey = "issuer-kind" + issuerGroupKey = "issuer-group" +) + +func (d *Driver) handleRuntimeConfigIssuerChange(logger logr.Logger, event watch.Event) error { + d.activeIssuerRefMutex.Lock() + defer d.activeIssuerRefMutex.Unlock() + + cm, ok := event.Object.(*corev1.ConfigMap) + if !ok { + return fmt.Errorf("got unexpected type for runtime configuration source; this is likely a programming error") + } + + issuerRef := &cmmeta.ObjectReference{} + + var dataErrs []error + var exists bool + + issuerRef.Name, exists = cm.Data[issuerNameKey] + if !exists || len(issuerRef.Name) == 0 { + dataErrs = append(dataErrs, fmt.Errorf("missing key/value in ConfigMap data: %s", issuerNameKey)) + } + + issuerRef.Kind, exists = cm.Data[issuerKindKey] + if !exists || len(issuerRef.Kind) == 0 { + dataErrs = append(dataErrs, fmt.Errorf("missing key/value in ConfigMap data: %s", issuerKindKey)) + } + + issuerRef.Group, exists = cm.Data[issuerGroupKey] + if !exists || len(issuerRef.Group) == 0 { + dataErrs = append(dataErrs, fmt.Errorf("missing key/value in ConfigMap data; %s", issuerGroupKey)) + } + + if len(dataErrs) > 0 { + return errors.Join(dataErrs...) + } + + // we now have a full issuerRef + // TODO: check if the issuer exists by querying for the CRD? + + d.activeIssuerRef = issuerRef + + logger.Info("Changed active issuerRef in response to runtime configuration ConfigMap", "issuer-name", d.activeIssuerRef.Name, "issuer-kind", d.activeIssuerRef.Kind, "issuer-group", d.activeIssuerRef.Group) + + return nil +} + +func (d *Driver) handleRuntimeConfigIssuerDeletion(logger logr.Logger) { + d.activeIssuerRefMutex.Lock() + defer d.activeIssuerRefMutex.Unlock() + + if d.originalIssuerRef == nil { + logger.Info("Runtime issuance configuration was deleted and no issuerRef was configured at install time; issuance will fail until runtime configuration is reinstated") + d.activeIssuerRef = nil + return + } + + logger.Info("Runtime issuance configuration was deleted; issuance will revert to original issuerRef configured at install time") + + d.activeIssuerRef = d.originalIssuerRef +} + +// Run is a blocking func that runs the CSI driver. func (d *Driver) Run(ctx context.Context) error { var wg sync.WaitGroup @@ -238,6 +443,12 @@ func (d *Driver) Run(ctx context.Context) error { d.camanager.run(ctx, updateRetryPeriod) }() + wg.Add(1) + go func() { + defer wg.Done() + d.watchRuntimeConfigurationSource(ctx) + }() + wg.Add(1) var err error go func() { @@ -252,6 +463,13 @@ func (d *Driver) Run(ctx context.Context) error { // generateRequest will generate a SPIFFE manager.CertificateRequestBundle // based upon the identity contained in the metadata service account token. func (d *Driver) generateRequest(meta metadata.Metadata) (*manager.CertificateRequestBundle, error) { + d.activeIssuerRefMutex.RLock() + defer d.activeIssuerRefMutex.RUnlock() + + if d.activeIssuerRef == nil { + return nil, fmt.Errorf("no issuerRef is currently active for csi-driver-spiffe; configure one using runtime configuration") + } + // Extract the service account token from the volume metadata in order to // derive the service account, and thus identity of the pod. token, err := util.EmptyAudienceTokenFromMetadata(meta) @@ -312,7 +530,7 @@ func (d *Driver) generateRequest(meta metadata.Metadata) (*manager.CertificateRe cmapi.UsageServerAuth, cmapi.UsageClientAuth, }, - IssuerRef: d.issuerRef, + IssuerRef: *d.activeIssuerRef, Annotations: crAnnotations, }, nil } @@ -377,3 +595,29 @@ func sanitizeAnnotations(in map[string]string) (map[string]string, error) { return out, errors.Join(errs...) } + +var errNoOriginalIssuer = fmt.Errorf("no original issuer was provided") + +func handleOriginalIssuerRef(in *cmmeta.ObjectReference) (*cmmeta.ObjectReference, error) { + if in == nil { + return nil, errNoOriginalIssuer + } + + if in.Name == "" && in.Kind == "" && in.Group == "" { + return nil, errNoOriginalIssuer + } + + if in.Name == "" { + return nil, fmt.Errorf("issuerRef.Name is a required field if any field is set for issuerRef") + } + + if in.Kind == "" { + return nil, fmt.Errorf("issuerRef.Kind is a required field if any field is set for issuerRef") + } + + if in.Group == "" { + return nil, fmt.Errorf("issuerRef.Group is a required field if any field is set for issuerRef") + } + + return in, nil +} diff --git a/make/test-e2e.mk b/make/test-e2e.mk index 5439a50..3eb1b8b 100644 --- a/make/test-e2e.mk +++ b/make/test-e2e.mk @@ -54,21 +54,23 @@ ifeq ($(findstring test-e2e,$(MAKECMDGOALS)),test-e2e) install: e2e-setup-example kind-cluster oci-load-manager oci-load-approver endif +E2E_RUNTIME_CONFIG_MAP_NAME ?= runtime-config-map +E2E_FOCUS ?= + test-e2e-deps: INSTALL_OPTIONS := test-e2e-deps: INSTALL_OPTIONS += --set image.repository.driver=$(oci_manager_image_name_development) test-e2e-deps: INSTALL_OPTIONS += --set image.repository.approver=$(oci_approver_image_name_development) test-e2e-deps: INSTALL_OPTIONS += --set image.pullPolicy=Never test-e2e-deps: INSTALL_OPTIONS += --set app.trustDomain=foo.bar -test-e2e-deps: INSTALL_OPTIONS += --set app.approver.signerName=clusterissuers.cert-manager.io/csi-driver-spiffe-ca test-e2e-deps: INSTALL_OPTIONS += --set app.issuer.name=csi-driver-spiffe-ca test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumes[0].name=root-cas test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumes[0].secret.secretName=csi-driver-spiffe-ca test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumeMounts[0].name=root-cas test-e2e-deps: INSTALL_OPTIONS += --set app.driver.volumeMounts[0].mountPath=/var/run/secrets/cert-manager-csi-driver-spiffe test-e2e-deps: INSTALL_OPTIONS += --set app.driver.sourceCABundle=/var/run/secrets/cert-manager-csi-driver-spiffe/ca.crt +test-e2e-deps: INSTALL_OPTIONS += --set app.runtimeIssuanceConfigMap=$(E2E_RUNTIME_CONFIG_MAP_NAME) test-e2e-deps: install -E2E_FOCUS ?= .PHONY: test-e2e ## e2e end-to-end tests @@ -82,4 +84,5 @@ test-e2e: test-e2e-deps | kind-cluster $(NEEDS_GINKGO) $(NEEDS_KUBECTL) $(ARTIFA -ldflags $(go_manager_ldflags) \ -- \ --kubeconfig-path=$(CURDIR)/$(kind_kubeconfig) \ - --kubectl-path=$(KUBECTL) + --kubectl-path=$(KUBECTL) \ + --runtime-issuance-config-map-name=$(E2E_RUNTIME_CONFIG_MAP_NAME) diff --git a/test/e2e/framework/config/config.go b/test/e2e/framework/config/config.go index d798a20..63b1c83 100644 --- a/test/e2e/framework/config/config.go +++ b/test/e2e/framework/config/config.go @@ -48,6 +48,9 @@ type Config struct { IssuerSecretName string RestConfig *rest.Config KubectlBinPath string + + IssuanceConfigMapName string + IssuanceConfigMapNamespace string } func (c *Config) AddFlags(fs *flag.FlagSet) *Config { @@ -89,5 +92,7 @@ func (c *Config) addFlags(fs *flag.FlagSet) *Config { fs.StringVar(&c.IssuerRef.Group, "issuer-group", "cert-manager.io", "Group of issuer which has been created for the test") fs.StringVar(&c.IssuerSecretName, "issuer-secret-name", "csi-driver-spiffe-ca", "Name of the CA certificate Secret") fs.StringVar(&c.IssuerSecretNamespace, "issuer-secret-namespace", "cert-manager", "Namespace where the CA certificate Secret is stored") + fs.StringVar(&c.IssuanceConfigMapName, "runtime-issuance-config-map-name", "runtime-config-map", "Name of runtime issuance ConfigMap") + fs.StringVar(&c.IssuanceConfigMapNamespace, "runtime-issuance-config-map-namespace", "cert-manager", "Namespace for runtime issuance ConfigMap") return c } diff --git a/test/e2e/suite/import.go b/test/e2e/suite/import.go index 8638cda..cc8760a 100644 --- a/test/e2e/suite/import.go +++ b/test/e2e/suite/import.go @@ -20,4 +20,5 @@ import ( _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/approval" _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/carotation" _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/fsgroup" + _ "github.com/cert-manager/csi-driver-spiffe/test/e2e/suite/runtimeconfiguration" ) diff --git a/test/e2e/suite/runtimeconfiguration/runtimeconfiguration.go b/test/e2e/suite/runtimeconfiguration/runtimeconfiguration.go new file mode 100644 index 0000000..32c7d27 --- /dev/null +++ b/test/e2e/suite/runtimeconfiguration/runtimeconfiguration.go @@ -0,0 +1,369 @@ +/* +Copyright 2024 The cert-manager Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtimeconfiguration + +import ( + "time" + + "github.com/cert-manager/cert-manager/pkg/util/pki" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + "github.com/cert-manager/csi-driver-spiffe/test/e2e/framework" + "github.com/cert-manager/csi-driver-spiffe/test/e2e/util" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ( + mountPath = "/var/run/secrets/my-pod" + containerName = "my-container" +) + +var _ = framework.CasesDescribe("RuntimeConfiguration", func() { + f := framework.NewDefaultFramework("RuntimeConfiguration") + + var ( + serviceAccount corev1.ServiceAccount + role rbacv1.Role + rolebinding rbacv1.RoleBinding + podTemplate corev1.Pod + ) + + JustBeforeEach(func() { + By("Creating test resources") + + serviceAccount = corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Namespace: f.Namespace.Name, GenerateName: "csi-driver-spiffe-e2e-sa-"}, + } + Expect(f.Client().Create(f.Context(), &serviceAccount)).NotTo(HaveOccurred()) + + role = rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "csi-driver-spiffe-e2e-role-", + Namespace: f.Namespace.Name, + }, + Rules: []rbacv1.PolicyRule{{ + Verbs: []string{"create"}, + APIGroups: []string{"cert-manager.io"}, + Resources: []string{"certificaterequests"}, + }}, + } + Expect(f.Client().Create(f.Context(), &role)).NotTo(HaveOccurred()) + + rolebinding = rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "csi-driver-spiffe-e2e-rolebinding-", + Namespace: f.Namespace.Name, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: serviceAccount.Name, + Namespace: f.Namespace.Name, + }}, + } + Expect(f.Client().Create(f.Context(), &rolebinding)).NotTo(HaveOccurred()) + + podTemplate = corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-pod-", + Namespace: f.Namespace.Name, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccount.Name, + Volumes: []corev1.Volume{{ + Name: "csi-driver-spiffe", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "spiffe.csi.cert-manager.io", + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{}, + }, + }, + }}, + Containers: []corev1.Container{ + { + Name: containerName, + Image: "docker.io/library/busybox:1.36.1-musl", + ImagePullPolicy: corev1.PullNever, + Command: []string{"sleep", "10000"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "csi-driver-spiffe", + MountPath: mountPath, + }, + }, + }, + }, + }, + } + + }) + + JustAfterEach(func() { + By("Cleaning up test resources") + Expect(f.Client().Delete(f.Context(), &rolebinding)).NotTo(HaveOccurred()) + Expect(f.Client().Delete(f.Context(), &role)).NotTo(HaveOccurred()) + Expect(f.Client().Delete(f.Context(), &serviceAccount)).NotTo(HaveOccurred()) + }) + + It("should succeed with a simple pod and no runtime configuration", func() { + pod := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &pod)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &pod)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &pod)).NotTo(HaveOccurred()) + + bundle, err := util.ReadCertFromMountPath(f, mountPath, pod.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + Expect(bundle.CheckNotEmpty()).NotTo(HaveOccurred()) + }) + + It("should succeed with a new issuer configured at runtime and revert when runtime configuration is deleted", func() { + // podOne should be created with the old issuer, since no ConfigMap has been created yet + By("Creating a pod before any runtime configuration") + podOne := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podOne)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podOne)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podOne)).NotTo(HaveOccurred()) + + By("Checking the pod used the issuer configured on startup") + cliArgCertBundle, err := util.ReadCertFromSecret(f, f.Config().IssuerSecretName, f.Config().IssuerSecretNamespace) + Expect(err).NotTo(HaveOccurred()) + + cliArgCert, err := pki.DecodeX509CertificateBytes(cliArgCertBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + podOneBundle, err := util.ReadCertFromMountPath(f, mountPath, podOne.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podOneChain, err := pki.DecodeX509CertificateChainBytes(podOneBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podOneChain[0].CheckSignatureFrom(cliArgCert)).NotTo(HaveOccurred()) + + By("Creating a new issuer to use at runtime") + issuerRef, newCABundle, cleanup, err := util.CreateNewCAIssuer(f) + defer func() { + err := cleanup() + Expect(err).NotTo(HaveOccurred()) + }() + + Expect(err).NotTo(HaveOccurred()) + + newCACert, err := pki.DecodeX509CertificateBytes(newCABundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + runtimeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Config().IssuanceConfigMapName, + Namespace: f.Config().IssuanceConfigMapNamespace, + }, + Data: map[string]string{ + "issuer-name": issuerRef.Name, + "issuer-kind": issuerRef.Kind, + "issuer-group": issuerRef.Group, + }, + } + + By("Creating runtime configuration to point at the new issuer") + Expect(f.Client().Create(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + + configMapNeedsCleanup := true + defer func() { + if configMapNeedsCleanup { + Expect(f.Client().Delete(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + } + }() + + By("Waiting a little for runtime configuration to propagate") + time.Sleep(5 * time.Second) + + // now we've created the runtime configuration configmap, a newly created pod should + // use the new issuer + + By("Creating a second pod after runtime configuration was created") + podTwo := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podTwo)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podTwo)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podTwo)).NotTo(HaveOccurred()) + + By("Checking that the second pod used the new issuer") + podTwoBundle, err := util.ReadCertFromMountPath(f, mountPath, podTwo.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podTwoChain, err := pki.DecodeX509CertificateChainBytes(podTwoBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podTwoChain[0].CheckSignatureFrom(cliArgCert)).To(HaveOccurred()) + Expect(podTwoChain[0].CheckSignatureFrom(newCACert)).NotTo(HaveOccurred()) + + By("Deleting the configuration ConfigMap") + Expect(f.Client().Delete(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + // we explicitly deleted the ConfigMap as part of the test - no need to clean it up any more + configMapNeedsCleanup = false + + By("Waiting a little for runtime configuration to revert") + time.Sleep(5 * time.Second) + + By("Creating a third pod after runtime configuration was deleted") + podThree := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podThree)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podThree)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podThree)).NotTo(HaveOccurred()) + + By("Checking that the third pod used the original issuer") + podThreeBundle, err := util.ReadCertFromMountPath(f, mountPath, podThree.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podThreeChain, err := pki.DecodeX509CertificateChainBytes(podThreeBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podThreeChain[0].CheckSignatureFrom(newCACert)).To(HaveOccurred()) + Expect(podThreeChain[0].CheckSignatureFrom(cliArgCert)).NotTo(HaveOccurred()) + }) + + It("should succeed with a new issuer configured at runtime and change issuers when configuration is updated", func() { + // First, fetch the cert for the CLI arg, to check later that it wasn't used to sign any pod certificates + cliArgCertBundle, err := util.ReadCertFromSecret(f, f.Config().IssuerSecretName, f.Config().IssuerSecretNamespace) + Expect(err).NotTo(HaveOccurred()) + + cliArgCert, err := pki.DecodeX509CertificateBytes(cliArgCertBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a new issuer to use at runtime") + issuerRefOne, newCABundleOne, cleanupIssuerOne, err := util.CreateNewCAIssuer(f) + defer func() { + err := cleanupIssuerOne() + Expect(err).NotTo(HaveOccurred()) + }() + + Expect(err).NotTo(HaveOccurred()) + + newCACertOne, err := pki.DecodeX509CertificateBytes(newCABundleOne.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + runtimeConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Config().IssuanceConfigMapName, + Namespace: f.Config().IssuanceConfigMapNamespace, + }, + Data: map[string]string{ + "issuer-name": issuerRefOne.Name, + "issuer-kind": issuerRefOne.Kind, + "issuer-group": issuerRefOne.Group, + }, + } + + By("Creating runtime configuration to point at the new issuer") + Expect(f.Client().Create(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + + defer func() { + Expect(f.Client().Delete(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + }() + + By("Waiting a little for runtime configuration to propagate") + time.Sleep(5 * time.Second) + + By("Creating a pod which uses runtime configuration") + podOne := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podOne)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podOne)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podOne)).NotTo(HaveOccurred()) + + By("Checking the pod used the new issuer") + podOneBundle, err := util.ReadCertFromMountPath(f, mountPath, podOne.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podOneChain, err := pki.DecodeX509CertificateChainBytes(podOneBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podOneChain[0].CheckSignatureFrom(cliArgCert)).To(HaveOccurred()) + Expect(podOneChain[0].CheckSignatureFrom(newCACertOne)).NotTo(HaveOccurred()) + + By("Creating a second new issuer to use at runtime") + issuerRefTwo, newCABundleTwo, cleanupIssuerTwo, err := util.CreateNewCAIssuer(f) + defer func() { + err := cleanupIssuerTwo() + Expect(err).NotTo(HaveOccurred()) + }() + + Expect(err).NotTo(HaveOccurred()) + + newCACertTwo, err := pki.DecodeX509CertificateBytes(newCABundleTwo.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + runtimeConfigMap.Data["issuer-name"] = issuerRefTwo.Name + runtimeConfigMap.Data["issuer-kind"] = issuerRefTwo.Kind + runtimeConfigMap.Data["issuer-group"] = issuerRefTwo.Group + + By("Updating runtime configuration to point at the new issuer") + Expect(f.Client().Update(f.Context(), runtimeConfigMap)).NotTo(HaveOccurred()) + + By("Waiting a little for the runtime configuration update to propagate") + time.Sleep(5 * time.Second) + + By("Creating a second pod after runtime configuration was updated") + podTwo := *podTemplate.DeepCopy() + + Expect(f.Client().Create(f.Context(), &podTwo)).NotTo(HaveOccurred()) + defer func() { + Expect(f.Client().Delete(f.Context(), &podTwo)).NotTo(HaveOccurred()) + }() + + Expect(util.WaitForPodReady(f, &podTwo)).NotTo(HaveOccurred()) + + By("Checking that the second pod used the new issuer") + podTwoBundle, err := util.ReadCertFromMountPath(f, mountPath, podTwo.Name, containerName) + Expect(err).NotTo(HaveOccurred()) + + podTwoChain, err := pki.DecodeX509CertificateChainBytes(podTwoBundle.CertificatePEM) + Expect(err).NotTo(HaveOccurred()) + + Expect(podTwoChain[0].CheckSignatureFrom(cliArgCert)).To(HaveOccurred()) + Expect(podTwoChain[0].CheckSignatureFrom(newCACertOne)).To(HaveOccurred()) + Expect(podTwoChain[0].CheckSignatureFrom(newCACertTwo)).NotTo(HaveOccurred()) + }) +}) diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 238e1e9..3daf8bc 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -23,9 +23,15 @@ import ( "fmt" "os/exec" "path/filepath" + "strings" "time" + cmapiutil "github.com/cert-manager/cert-manager/pkg/api/util" + cmapi "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8srand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -139,3 +145,265 @@ func ReadCertFromMountPath(f *framework.Framework, mountPath string, podName str return bundle, nil } + +// IssuerCleanupFunc is called to clean up issuer related resources after a test. Any returned +// cleanup function should always be safe to call and should always be called at some point after +// the returning function regardless of whether that function returned an error or not +type IssuerCleanupFunc func() error + +// dummyIssuerCleanupFunc should be returned by functions which return an IssuerCleanupFunc where +// there's nothing to clean up (e.g. if a fatal error happened before any resources were created). +func dummyIssuerCleanupFunc() error { + return nil +} + +// CreateSelfSignedIssuer creates a SelfSigned ClusterIssuer which can be used to in-turn create CA +// issuers for tests. +// Returns an issuerRef for the issuer and a cleanup function to remove the issuer after the test +// completes. +// The cleanup function is always safe to call and should always be called after this function +// returns, regardless of whether it returned an error or not +func CreateSelfSignedIssuer(f *framework.Framework) (*cmmeta.ObjectReference, IssuerCleanupFunc, error) { + iss := &cmapi.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "selfsigned-clusterissuer-", + }, + Spec: cmapi.IssuerSpec{ + IssuerConfig: cmapi.IssuerConfig{ + SelfSigned: &cmapi.SelfSignedIssuer{}, + }, + }, + } + + err := f.Client().Create(f.Context(), iss) + if err != nil { + return nil, dummyIssuerCleanupFunc, err + } + + cleanupFunc := func() error { + return f.Client().Delete(f.Context(), iss) + } + + issuerRef := &cmmeta.ObjectReference{ + Name: iss.Name, + Kind: "ClusterIssuer", + Group: "cert-manager.io", + } + + return issuerRef, cleanupFunc, nil +} + +// CreateNewCAIssuer creates an issuer which can be used for an end-to-end test and cleaned up +// afterwards. +// Returns an issuerRef for the issuer, a bundle containing the issuer's data and a function to +// clean up all issuer resources. +// The cleanup function is always safe to call and should always be called after this function +// returns, regardless of whether it returned an error or not +func CreateNewCAIssuer(f *framework.Framework) (*cmmeta.ObjectReference, *CertBundle, IssuerCleanupFunc, error) { + var objectsForCleanup []client.Object + + selfSignedIssuerRef, selfSignedCleanupFunc, err := CreateSelfSignedIssuer(f) + if err != nil { + return nil, nil, selfSignedCleanupFunc, fmt.Errorf("failed to create selfsigned ClusterIssuer: %s", err) + } + + cleanupFunc := func() error { + var errs []error + + selfSignedCleanupErr := selfSignedCleanupFunc() + if selfSignedCleanupErr != nil { + errs = append(errs, selfSignedCleanupErr) + } + + for _, m := range objectsForCleanup { + err := f.Client().Delete(f.Context(), m) + if err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) + } + + certSecretName := fmt.Sprintf("e2e-root-%s", k8srand.String(6)) + + cert := &cmapi.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: certSecretName, + Namespace: "cert-manager", // TODO: this might not always be the case + }, + Spec: cmapi.CertificateSpec{ + CommonName: certSecretName, + IsCA: true, + PrivateKey: &cmapi.CertificatePrivateKey{ + Algorithm: cmapi.ECDSAKeyAlgorithm, + Size: 256, + }, + SecretName: certSecretName, + IssuerRef: *selfSignedIssuerRef, + }, + } + + err = f.Client().Create(f.Context(), cert) + if err != nil { + return nil, nil, cleanupFunc, err + } + + objectsForCleanup = append(objectsForCleanup, cert) + + err = approveCertificateRequestsForCertificate(f, cert) + if err != nil { + return nil, nil, cleanupFunc, fmt.Errorf("failed to approve CertificateRequest: %s", err) + } + + err = WaitForCertificateReady(f, cert) + if err != nil { + return nil, nil, cleanupFunc, fmt.Errorf("failed to wait for cert to become ready: %s", err) + } + + iss := &cmapi.ClusterIssuer{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("ca-issuer-%s", certSecretName), + }, + Spec: cmapi.IssuerSpec{ + IssuerConfig: cmapi.IssuerConfig{ + CA: &cmapi.CAIssuer{ + SecretName: cert.Spec.SecretName, + }, + }, + }, + } + + err = f.Client().Create(f.Context(), iss) + if err != nil { + return nil, nil, cleanupFunc, err + } + + objectsForCleanup = append(objectsForCleanup, iss) + + caBundle, err := ReadCertFromSecret(f, certSecretName, "cert-manager") + if err != nil { + return nil, nil, cleanupFunc, err + } + + newIssuerRef := cmmeta.ObjectReference{ + Name: iss.Name, + Kind: "ClusterIssuer", + Group: "cert-manager.io", + } + + return &newIssuerRef, caBundle, cleanupFunc, nil +} + +// ReadCertFromSecret loads a certificate bundle from a Secret resource +func ReadCertFromSecret(f *framework.Framework, secretName string, secretNamespace string) (*CertBundle, error) { + certSecret := &corev1.Secret{} + + key := client.ObjectKey{ + Name: secretName, + Namespace: secretNamespace, + } + + err := f.Client().Get(f.Context(), key, certSecret) + if err != nil { + return nil, fmt.Errorf("failed to fetch secret %s/%s: %s", secretNamespace, secretName, err) + } + + chainBytes, exists := certSecret.Data["tls.crt"] + if !exists { + return nil, fmt.Errorf("failed to find certificate chain in secret %s/%s", secretNamespace, secretName) + } + + privateKeyBytes, exists := certSecret.Data["tls.key"] + if !exists { + return nil, fmt.Errorf("failed to find private key in secret %s/%s", secretNamespace, secretName) + } + + caBytes, exists := certSecret.Data["ca.crt"] + if !exists { + return nil, fmt.Errorf("failed to find CA data in secret %s/%s", secretNamespace, secretName) + } + + return &CertBundle{ + CertificatePEM: chainBytes, + PrivateKeyPEM: privateKeyBytes, + CAPEM: caBytes, + }, nil +} + +// WaitForCertificateReady waits until the references Certificate resource is marked as ready +func WaitForCertificateReady(f *framework.Framework, cert *cmapi.Certificate) error { + timeout := 60 * time.Second + interval := 2 * time.Second + immediate := false + + return wait.PollUntilContextTimeout(f.Context(), interval, timeout, immediate, func(ctx context.Context) (bool, error) { + err := f.Client().Get(ctx, client.ObjectKeyFromObject(cert), cert) + if err != nil { + return false, err + } + + for _, cond := range cert.Status.Conditions { + if cond.Type != cmapi.CertificateConditionReady { + continue + } + + return cond.Status == cmmeta.ConditionTrue, nil + } + + return false, nil + }) +} + +func approveCertificateRequestsForCertificate(f *framework.Framework, cert *cmapi.Certificate) error { + crList := &cmapi.CertificateRequestList{} + + listOpts := &client.ListOptions{ + Namespace: cert.Namespace, + } + + timeout := 10 * time.Second + interval := 1 * time.Second + immediate := false + + err := wait.PollUntilContextTimeout(f.Context(), interval, timeout, immediate, func(ctx context.Context) (bool, error) { + err := f.Client().List(ctx, crList, listOpts) + if err != nil { + return false, err + } + + for _, cr := range crList.Items { + if strings.HasPrefix(cr.Name, cert.Name) { + return true, nil + } + } + + return false, nil + }) + if err != nil { + return fmt.Errorf("failed to wait for CertificateRequest to be created: %s", err) + } + + var updateErrs []error + + for _, cr := range crList.Items { + cr := cr + + if !strings.HasPrefix(cr.Name, cert.Name) { + continue + } + + cmapiutil.SetCertificateRequestCondition(&cr, cmapi.CertificateRequestConditionApproved, cmmeta.ConditionTrue, "csi-driver-spiffe-e2e-test", "Manually approved for csi-driver-spiffe e2e tests") + + err := f.Client().Status().Update(f.Context(), &cr) + if err != nil { + updateErrs = append(updateErrs, err) + } + } + + if len(updateErrs) > 0 { + return errors.Join(updateErrs...) + } + + return nil +}