fix: make leaderelection possible

app-sre · Oct 27, 2022 · 52946de · 52946de
1 parent 5711726
commit 52946de
Show file tree

Hide file tree

Showing 13 changed files with 612 additions and 127 deletions.
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strconv"
@@ -18,7 +19,6 @@ import (
 	dvo_prom "github.com/app-sre/deployment-validation-operator/pkg/prometheus"
 	"github.com/app-sre/deployment-validation-operator/pkg/validations"
 	"github.com/app-sre/deployment-validation-operator/version"
-	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/go-logr/logr"
 	osappsv1 "github.com/openshift/api/apps/v1"
@@ -42,13 +42,18 @@ func main() {
 	os.Setenv(operatorNameEnvVar, dv_config.OperatorName)
 
 	opts := options{
-		MetricsPort: 8383,
-		MetricsPath: "metrics",
-		ProbeAddr:   ":8081",
-		ConfigFile:  "config/deployment-validation-operator-config.yaml",
+		MetricsBindAddr:    ":8383",
+		MetricsPath:        "metrics",
+		MetricsServiceName: "deployment-validation-operator-metrics",
+		ProbeAddr:          ":8081",
+		ConfigFile:         "config/deployment-validation-operator-config.yaml",
 	}
 
-	opts.Process()
+	if err := opts.Process(); err != nil {
+		fmt.Fprintf(os.Stdout, "processing options: %v\n", err)
+
+		os.Exit(1)
+	}
 
 	// Use a zap logr.Logger implementation. If none of the zap
 	// flags are configured (or if the zap flag set is not being
@@ -105,12 +110,8 @@ func setupManager(log logr.Logger, opts options) (manager.Manager, error) {
 		return nil, fmt.Errorf("initializing manager: %w", err)
 	}
 
-	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
-		return nil, fmt.Errorf("adding healthz check: %w", err)
-	}
-
-	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
-		return nil, fmt.Errorf("adding readyz check: %w", err)
+	if err := setupProbes(mgr, opts); err != nil {
+		return nil, fmt.Errorf("setting up probes: %w", err)
 	}
 
 	log.Info("Registering Components")
@@ -125,29 +126,12 @@ func setupManager(log logr.Logger, opts options) (manager.Manager, error) {
 		return nil, fmt.Errorf("initializing generic reconciler: %w", err)
 	}
 
-	if err = gr.AddToManager(mgr); err != nil {
+	if err := gr.AddToManager(mgr); err != nil {
 		return nil, fmt.Errorf("adding generic reconciler to manager: %w", err)
 	}
 
-	log.Info("Initializing Prometheus Registry")
-
-	reg := prometheus.NewRegistry()
-
-	log.Info(fmt.Sprintf("Initializing Prometheus metrics endpoint on %q", opts.MetricsEndpoint()))
-
-	srv, err := dvo_prom.NewServer(reg, opts.MetricsPath, fmt.Sprintf(":%d", opts.MetricsPort))
-	if err != nil {
-		return nil, fmt.Errorf("initializing metrics server: %w", err)
-	}
-
-	if err := mgr.Add(srv); err != nil {
-		return nil, fmt.Errorf("adding metrics server to manager: %w", err)
-	}
-
-	log.Info("Initializing Validation Engine")
-
-	if err := validations.InitializeValidationEngine(opts.ConfigFile, reg); err != nil {
-		return nil, fmt.Errorf("initializing validation engine: %w", err)
+	if err := setupComponents(log, mgr, opts); err != nil {
+		return nil, fmt.Errorf("setting up components: %w", err)
 	}
 
 	return mgr, nil
@@ -192,9 +176,13 @@ func getManagerOptions(scheme *k8sruntime.Scheme, opts options) (manager.Options
 	}
 
 	mgrOpts := manager.Options{
-		Namespace:              ns,
-		HealthProbeBindAddress: opts.ProbeAddr,
-		MetricsBindAddress:     "0", // disable controller-runtime managed prometheus endpoint
+		LeaderElection:             opts.EnableLeaderElection,
+		LeaderElectionID:           "23h85e23.deployment-validation-operator-lock",
+		LeaderElectionNamespace:    opts.LeaderElectionNamespace,
+		LeaderElectionResourceLock: "leases",
+		Namespace:                  ns,
+		HealthProbeBindAddress:     opts.ProbeAddr,
+		MetricsBindAddress:         "0", // disable controller-runtime managed prometheus endpoint
 		// disable caching of everything
 		NewClient: newClient,
 		Scheme:    scheme,
@@ -236,3 +224,62 @@ func kubeClientQPS() (float32, error) {
 	qps = float32(val)
 	return qps, err
 }
+
+func setupProbes(mgr manager.Manager, opts options) error {
+	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		return fmt.Errorf("adding healthz check: %w", err)
+	}
+
+	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		return fmt.Errorf("adding readyz check: %w", err)
+	}
+
+	return nil
+}
+
+func setupComponents(log logr.Logger, mgr manager.Manager, opts options) error {
+	log.Info("Initializing Prometheus Registry")
+
+	reg, err := dvo_prom.NewRegistry()
+	if err != nil {
+		return fmt.Errorf("initializing prometheus registry: %w", err)
+	}
+
+	log.Info(fmt.Sprintf("Initializing Prometheus metrics endpoint on %q", opts.MetricsEndpoint()))
+
+	svcURL := &url.URL{
+		Scheme: "http",
+		Host:   opts.MetricsServiceName,
+	}
+	if parts := strings.Split(opts.MetricsBindAddr, ":"); len(parts) > 0 {
+		if len(parts) > 1 {
+			svcURL.Host += parts[len(parts)-1]
+		}
+	}
+
+	srv, err := dvo_prom.NewServer(reg,
+		dvo_prom.WithMetricsAddr(opts.MetricsBindAddr),
+		dvo_prom.WithMetricsPath(opts.MetricsPath),
+		dvo_prom.WithServiceURL(svcURL.String()),
+	)
+	if err != nil {
+		return fmt.Errorf("initializing metrics server: %w", err)
+	}
+
+	go func() {
+		<-mgr.Elected()
+		srv.Ready()
+	}()
+
+	if err := mgr.Add(srv); err != nil {
+		return fmt.Errorf("adding metrics server to manager: %w", err)
+	}
+
+	log.Info("Initializing Validation Engine")
+
+	if err := validations.InitializeValidationEngine(opts.ConfigFile, reg); err != nil {
+		return fmt.Errorf("initializing validation engine: %w", err)
+	}
+
+	return nil
+}
diff --git a/cmd/manager/options.go b/cmd/manager/options.go
@@ -1,25 +1,42 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
+	"net/url"
 	"os"
+	"strings"
 
 	"github.com/spf13/pflag"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
 
 type options struct {
-	MetricsPort    int32
-	MetricsPath    string
-	ProbeAddr      string
-	ConfigFile     string
-	watchNamespace *string
-	Zap            zap.Options
+	EnableLeaderElection    bool
+	LeaderElectionNamespace string
+	MetricsBindAddr         string
+	MetricsPath             string
+	MetricsServiceName      string
+	ProbeAddr               string
+	ConfigFile              string
+	watchNamespace          *string
+	Zap                     zap.Options
 }
 
 func (o *options) MetricsEndpoint() string {
-	return fmt.Sprintf("http://0.0.0.0:%d/%s", o.MetricsPort, o.MetricsPath)
+	endpoint := &url.URL{
+		Scheme: "http",
+		Path:   o.MetricsPath,
+	}
+
+	if addr := o.MetricsBindAddr; strings.HasPrefix(addr, ":") {
+		endpoint.Host = "0.0.0.0" + o.MetricsBindAddr
+	} else {
+		endpoint.Host = o.MetricsBindAddr
+	}
+
+	return endpoint.String()
 }
 
 func (o *options) GetWatchNamespace() (string, bool) {
@@ -30,9 +47,16 @@ func (o *options) GetWatchNamespace() (string, bool) {
 	return *o.watchNamespace, true
 }
 
-func (o *options) Process() {
+func (o *options) Process() error {
 	o.processFlags()
 	o.processEnv()
+	o.processSecrets()
+
+	if err := o.validate(); err != nil {
+		return fmt.Errorf("validating options: %w", err)
+	}
+
+	return nil
 }
 
 func (o *options) processFlags() {
@@ -51,11 +75,31 @@ func (o *options) processFlags() {
 		"config", o.ConfigFile,
 		"Path to config file",
 	)
+	flags.BoolVar(
+		&o.EnableLeaderElection,
+		"enable-leader-election", o.EnableLeaderElection,
+		"Enables Leader Election when starting the manager.",
+	)
+	flags.StringVar(
+		&o.LeaderElectionNamespace,
+		"leader-election-namespace", o.LeaderElectionNamespace,
+		"The namespace used by leader election resources.",
+	)
+	flags.StringVar(
+		&o.MetricsBindAddr,
+		"metrics-bind-address", o.MetricsBindAddr,
+		"The address the metrics endpoint binds to.",
+	)
 	flags.StringVar(
 		&o.ProbeAddr,
 		"health-probe-bind-address", o.ProbeAddr,
 		"The address the probe endpoint binds to.",
 	)
+	flags.StringVar(
+		&o.MetricsServiceName,
+		"metrics-service-name", o.MetricsServiceName,
+		"Name of the service used to load balance metrics",
+	)
 
 	pflag.CommandLine.AddFlagSet(flags)
 
@@ -72,3 +116,35 @@ func (o *options) processEnv() {
 		o.watchNamespace = &val
 	}
 }
+
+func (o *options) processSecrets() {
+	const (
+		scrtsPath              = "/var/run/secrets"
+		inClusterNamespacePath = scrtsPath + "/kubernetes.io/serviceaccount/namespace"
+	)
+
+	var namespace string
+
+	if ns, err := os.ReadFile(inClusterNamespacePath); err == nil {
+		// Avoid applying a garbage value if an error occurred
+		namespace = string(ns)
+	}
+
+	if o.LeaderElectionNamespace == "" {
+		o.LeaderElectionNamespace = namespace
+	}
+}
+
+var errLeaderElectionNamespaceNotSet = errors.New("leader election namespace not set")
+
+func (o *options) validate() error {
+	if !o.EnableLeaderElection {
+		return nil
+	}
+
+	if o.LeaderElectionNamespace != "" {
+		return nil
+	}
+
+	return errLeaderElectionNamespaceNotSet
+}
diff --git a/go.mod b/go.mod
@@ -9,11 +9,13 @@ require (
 	github.com/mcuadros/go-defaults v1.2.0
 	github.com/openshift/api v3.9.0+incompatible
 	github.com/prometheus/client_golang v1.12.0
-	github.com/prometheus/client_model v0.2.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.9.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	go.uber.org/multierr v1.6.0
+	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 // indirect
 	golang.stackrox.io/kube-linter v0.0.0-20210928184316-5e1ead387f43
 	k8s.io/api v0.22.2
 	k8s.io/apimachinery v0.22.2

diff --git a/go.sum b/go.sum
@@ -1093,8 +1093,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -1248,8 +1249,9 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1345,8 +1347,10 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1478,21 +1482,25 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7 h1:8IVLkfbr2cLhv0a/vKq4UFUcJym8RmDoDboxCFWEjYE=
+golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=