Add extraArgs config for kube-router

This is for advanced use cases and users that need to control things like ASN, routing and peering. Allows also users to override any k0s managed startup args.

Signed-off-by: Jussi Nummelin <[email protected]>

docs/configuration.md

@@ -91,6 +91,8 @@ spec:
       mtu: 0
       peerRouterASNs: ""
       peerRouterIPs: ""
+      extraArgs:
+        - foo=bar
     nodeLocalLoadBalancing:
       enabled: false
       envoyProxy:
@@ -213,15 +215,16 @@ CALICO_IPV6POOL_CIDR: "{{ spec.network.dualStack.IPv6podCIDR }}"
 #### `spec.network.kuberouter`
 
 | Element          | Description                                                                                                                                                                                                                                                               |
-| ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `autoMTU`        | Autodetection of used MTU (default: `true`).                                                                                                                                                                                                                              |
 | `mtu`            | Override MTU setting, if `autoMTU` must be set to `false`).                                                                                                                                                                                                               |
 | `metricsPort`    | Kube-router metrics server port. Set to 0 to disable metrics  (default: `8080`).                                                                                                                                                                                          |
 | `peerRouterIPs`  | Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                                                                        |
 | `peerRouterASNs` | Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                                                                             |
 | `hairpin`        | Hairpin mode, supported modes `Enabled`: enabled cluster wide, `Allowed`: must be allowed per service [using annotations](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode), `Disabled`: doesn't work at all (default: Enabled) |
 | `hairpinMode`    | **Deprecated** Use `hairpin` instead. If both `hairpin` and `hairpinMode` are defined, this is ignored. If only hairpinMode is configured explicitly activates hairpinMode (https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode).  |
-| `ipMasq`         | IP masquerade for traffic originating from the pod network, and destined outside of it (default: false) |
+| `ipMasq`         | IP masquerade for traffic originating from the pod network, and destined outside of it (default: false)                                                                                                                                                                   |
+| `extraArgs`      | Extra arguments to pass to kube-router. Can be also used to override any k0s managed args. For reference, see kube-router [documentation](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#command-line-options). (default: empty)                             |
 
 **Note**: Kube-router allows many networking aspects to be configured per node, service, and pod (for more information, refer to the [Kube-router user guide](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md)).
 

pkg/apis/k0s/v1beta1/kuberouter.go

@@ -36,6 +36,9 @@ type KubeRouter struct {
 	PeerRouterASNs string `json:"peerRouterASNs"`
 	// Comma-separated list of global peer ASNs
 	PeerRouterIPs string `json:"peerRouterIPs"`
+	// ExtraArgs are extra arguments to pass to kube-router
+	// Can be also used to override the default k0s managed kube-router arguments
+	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Enabled;Allowed;Disabled

pkg/component/controller/kuberouter.go

@@ -20,7 +20,9 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"reflect"
 
+	"github.com/k0sproject/k0s/internal/pkg/stringmap"
 	"github.com/k0sproject/k0s/internal/pkg/templatewriter"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/component/manager"
@@ -55,6 +57,7 @@ type kubeRouterConfig struct {
 	PeerRouterIPs     string
 	PeerRouterASNs    string
 	PullPolicy        string
+	Args              []string
 }
 
 // NewKubeRouter creates new KubeRouter reconciler component
@@ -73,25 +76,26 @@ func (k *KubeRouter) Init(_ context.Context) error { return nil }
 // Stop no-op as nothing running
 func (k *KubeRouter) Stop() error { return nil }
 
-func getHairpinConfig(cfg *kubeRouterConfig, krc *v1beta1.KubeRouter) {
+func getHairpinConfig(krc *v1beta1.KubeRouter) (cniHairpin bool, globalHairpin bool) {
 	// Configure hairpin
 	switch krc.Hairpin {
 	case v1beta1.HairpinUndefined:
 		// If Hairpin is undefined, then we honor HairpinMode
 		if krc.HairpinMode {
-			cfg.CNIHairpin = true
-			cfg.GlobalHairpin = true
+			cniHairpin = true
+			globalHairpin = true
 		}
 	case v1beta1.HairpinDisabled:
-		cfg.CNIHairpin = false
-		cfg.GlobalHairpin = false
+		cniHairpin = false
+		globalHairpin = false
 	case v1beta1.HairpinAllowed:
-		cfg.CNIHairpin = true
-		cfg.GlobalHairpin = false
+		cniHairpin = true
+		globalHairpin = false
 	case v1beta1.HairpinEnabled:
-		cfg.CNIHairpin = true
-		cfg.GlobalHairpin = true
+		cniHairpin = true
+		globalHairpin = true
 	}
+	return
 }
 
 // Reconcile detects changes in configuration and applies them to the component
@@ -106,20 +110,44 @@ func (k *KubeRouter) Reconcile(_ context.Context, clusterConfig *v1beta1.Cluster
 		return fmt.Errorf("cannot change CNI provider from %s to %s", existingCNI, constant.CNIProviderKubeRouter)
 	}
 
+	cniHairpin, globalHairpin := getHairpinConfig(clusterConfig.Spec.Network.KubeRouter)
+
+	args := stringmap.StringMap{
+		// Default args
+		"run-router":           "true",
+		"run-firewall":         "true",
+		"run-service-proxy":    "false",
+		"bgp-graceful-restart": "true",
+		// Configurable args
+		"auto-mtu":     fmt.Sprintf("%t", clusterConfig.Spec.Network.KubeRouter.AutoMTU),
+		"metrics-port": fmt.Sprintf("%d", clusterConfig.Spec.Network.KubeRouter.MetricsPort),
+		"hairpin-mode": fmt.Sprintf("%t", globalHairpin),
+	}
+
+	// We should not add peering flags if the values are empty
+	if clusterConfig.Spec.Network.KubeRouter.PeerRouterASNs != "" {
+		args["peer-router-asns"] = clusterConfig.Spec.Network.KubeRouter.PeerRouterASNs
+	}
+	if clusterConfig.Spec.Network.KubeRouter.PeerRouterIPs != "" {
+		args["peer-router-ips"] = clusterConfig.Spec.Network.KubeRouter.PeerRouterIPs
+	}
+
+	// Override or add args from config
+	args.Merge(clusterConfig.Spec.Network.KubeRouter.ExtraArgs)
+
 	cfg := kubeRouterConfig{
 		AutoMTU:           clusterConfig.Spec.Network.KubeRouter.AutoMTU,
 		MTU:               clusterConfig.Spec.Network.KubeRouter.MTU,
 		MetricsPort:       clusterConfig.Spec.Network.KubeRouter.MetricsPort,
-		PeerRouterIPs:     clusterConfig.Spec.Network.KubeRouter.PeerRouterIPs,
-		PeerRouterASNs:    clusterConfig.Spec.Network.KubeRouter.PeerRouterASNs,
 		IPMasq:            clusterConfig.Spec.Network.KubeRouter.IPMasq,
+		CNIHairpin:        cniHairpin,
 		CNIImage:          clusterConfig.Spec.Images.KubeRouter.CNI.URI(),
 		CNIInstallerImage: clusterConfig.Spec.Images.KubeRouter.CNIInstaller.URI(),
 		PullPolicy:        clusterConfig.Spec.Images.DefaultPullPolicy,
+		Args:              args.ToDashedArgs(),
 	}
-	getHairpinConfig(&cfg, clusterConfig.Spec.Network.KubeRouter)
 
-	if cfg == k.previousConfig {
+	if reflect.DeepEqual(k.previousConfig, cfg) {
 		k.log.Info("config matches with previous, not reconciling anything")
 		return nil
 	}
@@ -278,20 +306,8 @@ spec:
         image: {{ .CNIImage }}
         imagePullPolicy: {{ .PullPolicy }}
         args:
-        - "--run-router=true"
-        - "--run-firewall=true"
-        - "--run-service-proxy=false"
-        - "--bgp-graceful-restart=true"
-        - "--metrics-port={{ .MetricsPort }}"
-        - "--hairpin-mode={{ .GlobalHairpin }}"
-        {{- if not .AutoMTU }}
-        - "--auto-mtu=false"
-        {{- end }}
-        {{- if .PeerRouterIPs }}
-        - "--peer-router-ips={{ .PeerRouterIPs }}"
-        {{- end }}
-        {{- if .PeerRouterASNs }}
-        - "--peer-router-asns={{ .PeerRouterASNs }}"
+        {{- range .Args }}
+        - {{ . }}
         {{- end }}
         env:
         - name: NODE_NAME

pkg/component/controller/kuberouter_test.go

@@ -107,8 +107,8 @@ func TestGetHairpinConfig(t *testing.T) {
 
 	for _, test := range hairpinTests {
 		cfg := &kubeRouterConfig{}
-		getHairpinConfig(cfg, test.krc)
-		if cfg.CNIHairpin != test.result.CNIHairpin || cfg.GlobalHairpin != test.result.GlobalHairpin {
+		cniHairpin, globalHairpin := getHairpinConfig(test.krc)
+		if cniHairpin != test.result.CNIHairpin || globalHairpin != test.result.GlobalHairpin {
 			t.Fatalf("Hairpin configuration (%#v) does not match exepected output (%#v) ", cfg, test.result)
 		}
 	}
@@ -182,6 +182,38 @@ func TestKubeRouterManualMTUManifests(t *testing.T) {
 	require.Equal(t, float64(1234), p.Dig("mtu"))
 }
 
+func TestExtraArgs(t *testing.T) {
+	k0sVars, err := config.NewCfgVars(nil, t.TempDir())
+	require.NoError(t, err)
+	cfg := v1beta1.DefaultClusterConfig()
+	cfg.Spec.Network.Calico = nil
+	cfg.Spec.Network.Provider = "kuberouter"
+	cfg.Spec.Network.KubeRouter = v1beta1.DefaultKubeRouter()
+	cfg.Spec.Network.KubeRouter.ExtraArgs = map[string]string{
+		// Add some random arg
+		"foo": "bar",
+		// Override the default arg
+		"run-firewall": "false",
+	}
+
+	saver := inMemorySaver{}
+	kr := NewKubeRouter(k0sVars, saver)
+	require.NoError(t, kr.Reconcile(context.Background(), cfg))
+	require.NoError(t, kr.Stop())
+
+	manifestData, foundRaw := saver["kube-router.yaml"]
+	require.True(t, foundRaw, "must have manifests for kube-router")
+
+	resources, err := testutil.ParseManifests(manifestData)
+	require.NoError(t, err)
+	ds, err := findDaemonset(resources)
+	require.NoError(t, err)
+	require.NotNil(t, ds)
+
+	assert.Contains(t, ds.Spec.Template.Spec.Containers[0].Args, "--run-firewall=false")
+	assert.Contains(t, ds.Spec.Template.Spec.Containers[0].Args, "--foo=bar")
+}
+
 func findConfig(resources []*unstructured.Unstructured) (corev1.ConfigMap, error) {
 	var cm corev1.ConfigMap
 	for _, r := range resources {

static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml

@@ -428,6 +428,12 @@ spec:
                       autoMTU:
                         description: 'Auto-detection of used MTU (default: true)'
                         type: boolean
+                      extraArgs:
+                        additionalProperties:
+                          type: string
+                        description: ExtraArgs are extra arguments to pass to kube-router
+                          Can be also used to override the default kube-router arguments
+                        type: object
                       hairpin:
                         default: Enabled
                         description: 'Admits three values: "Enabled" enables it globally,