From 7fbbccb216a0ab6b4f79e009029134c7e340b9a4 Mon Sep 17 00:00:00 2001 From: Ben Ash Date: Mon, 25 Nov 2024 21:04:12 +0000 Subject: [PATCH] WIP: Add registry metrics collectors --- controllers/hcpvaultsecretsapp_controller.go | 8 ++ controllers/registry.go | 79 ++++++++++++++++++++ controllers/registry_test.go | 2 +- controllers/vaultauth_controller.go | 5 ++ controllers/vaultdynamicsecret_controller.go | 9 ++- controllers/vaultpkisecret_controller.go | 8 ++ controllers/vaultstaticsecret_controller.go | 9 +++ 7 files changed, 118 insertions(+), 2 deletions(-) diff --git a/controllers/hcpvaultsecretsapp_controller.go b/controllers/hcpvaultsecretsapp_controller.go index b9392e696..6cad5f56d 100644 --- a/controllers/hcpvaultsecretsapp_controller.go +++ b/controllers/hcpvaultsecretsapp_controller.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/hashicorp/vault-secrets-operator/credentials" "github.com/hashicorp/vault-secrets-operator/credentials/hcp" @@ -302,9 +303,16 @@ func (r *HCPVaultSecretsAppReconciler) updateStatus(ctx context.Context, o *secr // SetupWithManager sets up the controller with the Manager. func (r *HCPVaultSecretsAppReconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error { r.referenceCache = newResourceReferenceCache() + ctrlmetrics.Registry.MustRegister( + newResourceReferenceCacheCollector(r.referenceCache, "hcpvaultsecretsapp"), + ) + if r.BackOffRegistry == nil { r.BackOffRegistry = NewBackOffRegistry() } + ctrlmetrics.Registry.MustRegister( + newBackoffRegistryCacheCollector(r.BackOffRegistry, "hcpvaultsecretsapp"), + ) return ctrl.NewControllerManagedBy(mgr). For(&secretsv1beta1.HCPVaultSecretsApp{}). diff --git a/controllers/registry.go b/controllers/registry.go index 1f5fb819a..609287814 100644 --- a/controllers/registry.go +++ b/controllers/registry.go @@ -8,7 +8,10 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/prometheus/client_golang/prometheus" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/hashicorp/vault-secrets-operator/internal/metrics" ) type ResourceKind int @@ -49,6 +52,7 @@ type ResourceReferenceCache interface { Get(ResourceKind, client.ObjectKey) []client.ObjectKey Remove(ResourceKind, client.ObjectKey) bool Prune(ResourceKind, client.ObjectKey) int + Len() int } var _ ResourceReferenceCache = (*resourceReferenceCache)(nil) @@ -80,6 +84,20 @@ type resourceReferenceCache struct { mu sync.RWMutex } +func (c *resourceReferenceCache) Len() int { + c.mu.RLock() + defer c.mu.RUnlock() + + var l int + for _, scope := range c.m { + for _, refs := range scope { + l += len(refs) + } + } + + return l +} + // Set references of kind for referrer. func (c *resourceReferenceCache) Set(kind ResourceKind, referrer client.ObjectKey, references ...client.ObjectKey) { c.mu.Lock() @@ -274,6 +292,13 @@ func (r *BackOffRegistry) Get(objKey client.ObjectKey) (*BackOff, bool) { return entry, !ok } +func (r *BackOffRegistry) Len() int { + r.mu.RLock() + defer r.mu.RUnlock() + + return len(r.m) +} + // BackOff is a wrapper around backoff.BackOff that does not implement // BackOff.Reset, since elements in BackOffRegistry are meant to be ephemeral. type BackOff struct { @@ -304,3 +329,57 @@ func NewBackOffRegistry(opts ...backoff.ExponentialBackOffOpts) *BackOffRegistry opts: opts, } } + +// registryCacheCollector provides a prometheus.Collector for ClientCache metrics. +type registryCacheCollector struct { + cache ResourceReferenceCache + size float64 + lenDesc *prometheus.Desc +} + +func (c registryCacheCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.lenDesc +} + +func (c registryCacheCollector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric(c.lenDesc, prometheus.GaugeValue, float64(c.cache.Len())) +} + +func newResourceReferenceCacheCollector(cache ResourceReferenceCache, name string) prometheus.Collector { + metricsFQNClientCacheLength := prometheus.BuildFQName( + metrics.Namespace, "rsc_ref_cache", metrics.NameLength) + return ®istryCacheCollector{ + cache: cache, + lenDesc: prometheus.NewDesc( + metricsFQNClientCacheLength, + "Number of object references in the cache.", + nil, map[string]string{"name": name}), + } +} + +// backoffRegistryCacheCollector +type backoffRegistryCacheCollector struct { + cache *BackOffRegistry + size float64 + lenDesc *prometheus.Desc +} + +func (c backoffRegistryCacheCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.lenDesc +} + +func (c backoffRegistryCacheCollector) Collect(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric(c.lenDesc, prometheus.GaugeValue, float64(c.cache.Len())) +} + +func newBackoffRegistryCacheCollector(cache *BackOffRegistry, name string) prometheus.Collector { + metricsFQNClientCacheLength := prometheus.BuildFQName( + metrics.Namespace, "backoff_registry_cache", metrics.NameLength) + return &backoffRegistryCacheCollector{ + cache: cache, + lenDesc: prometheus.NewDesc( + metricsFQNClientCacheLength, + "Number of backoff objects in the cache.", + nil, map[string]string{"name": name}), + } +} diff --git a/controllers/registry_test.go b/controllers/registry_test.go index 1dedbffc7..f250a9014 100644 --- a/controllers/registry_test.go +++ b/controllers/registry_test.go @@ -60,7 +60,7 @@ func Test_resourceReferenceCache(t *testing.T) { { action: 0, referrer: referrer1, - references: []client.ObjectKey{reference1}, + references: []client.ObjectKey{reference1, reference1, reference1}, }, }, m: refCacheMap{ diff --git a/controllers/vaultauth_controller.go b/controllers/vaultauth_controller.go index 524c526f4..a7873d40a 100644 --- a/controllers/vaultauth_controller.go +++ b/controllers/vaultauth_controller.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/predicate" secretsv1beta1 "github.com/hashicorp/vault-secrets-operator/api/v1beta1" @@ -241,6 +242,10 @@ func (r *VaultAuthReconciler) handleFinalizer(ctx context.Context, o *secretsv1b // SetupWithManager sets up the controller with the Manager. func (r *VaultAuthReconciler) SetupWithManager(mgr ctrl.Manager) error { r.referenceCache = newResourceReferenceCache() + ctrlmetrics.Registry.MustRegister( + newResourceReferenceCacheCollector(r.referenceCache, "vaultauth"), + ) + return ctrl.NewControllerManagedBy(mgr). For(&secretsv1beta1.VaultAuth{}). Watches( diff --git a/controllers/vaultdynamicsecret_controller.go b/controllers/vaultdynamicsecret_controller.go index 048d75ecc..6cf77dbd4 100644 --- a/controllers/vaultdynamicsecret_controller.go +++ b/controllers/vaultdynamicsecret_controller.go @@ -22,9 +22,10 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/builder" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -596,9 +597,15 @@ func (r *VaultDynamicSecretReconciler) renewLease( // SetupWithManager sets up the controller with the Manager. func (r *VaultDynamicSecretReconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error { r.referenceCache = newResourceReferenceCache() + ctrlmetrics.Registry.MustRegister( + newResourceReferenceCacheCollector(r.referenceCache, "vaultdynamicsecret"), + ) if r.BackOffRegistry == nil { r.BackOffRegistry = NewBackOffRegistry() } + ctrlmetrics.Registry.MustRegister( + newBackoffRegistryCacheCollector(r.BackOffRegistry, "vaultdynamicsecret"), + ) r.ClientFactory.RegisterClientCallbackHandler( vault.ClientCallbackHandler{ diff --git a/controllers/vaultpkisecret_controller.go b/controllers/vaultpkisecret_controller.go index b34e3f452..b46820c87 100644 --- a/controllers/vaultpkisecret_controller.go +++ b/controllers/vaultpkisecret_controller.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" secretsv1beta1 "github.com/hashicorp/vault-secrets-operator/api/v1beta1" "github.com/hashicorp/vault-secrets-operator/consts" @@ -347,8 +348,15 @@ func (r *VaultPKISecretReconciler) handleDeletion(ctx context.Context, o *secret func (r *VaultPKISecretReconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error { r.referenceCache = newResourceReferenceCache() + ctrlmetrics.Registry.MustRegister( + newResourceReferenceCacheCollector(r.referenceCache, "vaultpkisecret"), + ) + if r.BackOffRegistry == nil { r.BackOffRegistry = NewBackOffRegistry() + ctrlmetrics.Registry.MustRegister( + newBackoffRegistryCacheCollector(r.BackOffRegistry, "vaultpkisecret"), + ) } return ctrl.NewControllerManagedBy(mgr). For(&secretsv1beta1.VaultPKISecret{}). diff --git a/controllers/vaultstaticsecret_controller.go b/controllers/vaultstaticsecret_controller.go index 92ec89225..66e1ccb27 100644 --- a/controllers/vaultstaticsecret_controller.go +++ b/controllers/vaultstaticsecret_controller.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/cenkalti/backoff/v4" @@ -500,9 +501,17 @@ func (r *VaultStaticSecretReconciler) streamStaticSecretEvents(ctx context.Conte func (r *VaultStaticSecretReconciler) SetupWithManager(mgr ctrl.Manager, opts controller.Options) error { r.referenceCache = newResourceReferenceCache() + ctrlmetrics.Registry.MustRegister( + newResourceReferenceCacheCollector(r.referenceCache, "vaultstaticsecret"), + ) + if r.BackOffRegistry == nil { r.BackOffRegistry = NewBackOffRegistry() } + ctrlmetrics.Registry.MustRegister( + newBackoffRegistryCacheCollector(r.BackOffRegistry, "vaultstaticsecret"), + ) + r.SourceCh = make(chan event.GenericEvent) r.eventWatcherRegistry = newEventWatcherRegistry()