Cherry picks v1.89.4 (#78)

* Remove Dependency of OperatingSystemConfig on Extensions (#40)

* Increase client-side rate limits of seed resource-manager

Signed-off-by: Niclas Schad <[email protected]>

* Use SKE etcd-druid (adds stackit S3 provider support)

* [manual:component:github.com/gardener/dependency-watchdog:v1.1.2->v1.2.1] (gardener#9072)

* Vendor new dwd release and create MR to create role and rolebinding in shoot

* run make format

* added unit tests

* Fix unit tests

* Update gardenlet to add dwd resources to shoot ns before starting controllers

* run make generate

* run make check

* Addressed review comments

* Address review comments

* run make generate

* remove nodeMonitoringGraceDuration from prober config

* Vendor dwd v1.2.1

* Added new DWDAccess interface

* Update comments

* Addresses review comments

* FIx dependency-watchdog permissions (gardener#9499)

* FIx dependency-watchdog permissions

Allow dwd to get/list nodes in the shoots

* Address review feedback

* update dependency-watchdog version v1.2.3

Signed-off-by: Niclas Schad <[email protected]>

* Fix fetching CA bundle secret in DWD migration function (gardener#9276)

* Enable AlwaysUpdate for resource manager targeting runtime cluster

* [ske-v1.88] cherrypick apiserver-proxy changes

Configure apiserver proxy to use HTTP connect

This reuses the reversed VPN listener and filter mechanisms on the istio-ingressgateway side.

Co-Authored-By: Johannes Scheerer <[email protected]>

Bump ext-authz-server

Add `DisableAPIServerProxyPort` feature gate

* Fix `Secret etcd-backup not found` (gardener#9800)

* Add deployNamespace dependency to deployBackupEntryInGarden

* Remove comment

* Only specify args in etcd-druid

---------

Signed-off-by: Niclas Schad <[email protected]>
Co-authored-by: Niclas Schad <[email protected]>
Co-authored-by: Tim Ebert <[email protected]>
Co-authored-by: Aaron Francis Fernandes <[email protected]>
Co-authored-by: Vladimir Nachev <[email protected]>
Co-authored-by: Rafael Franzke <[email protected]>
Co-authored-by: Viktor <[email protected]>

cmd/gardenlet/app/app.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"os"
 	goruntime "runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -33,12 +34,19 @@ import (
 	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
 	eventsv1 "k8s.io/api/events/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/runtime/serializer/json"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
@@ -65,6 +73,7 @@ import (
 	resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
 	"github.com/gardener/gardener/pkg/client/kubernetes"
 	clientmapbuilder "github.com/gardener/gardener/pkg/client/kubernetes/clientmap/builder"
+	dwd "github.com/gardener/gardener/pkg/component/dependencywatchdog"
 	"github.com/gardener/gardener/pkg/controllerutils"
 	"github.com/gardener/gardener/pkg/controllerutils/routes"
 	"github.com/gardener/gardener/pkg/features"
@@ -74,6 +83,7 @@ import (
 	"github.com/gardener/gardener/pkg/gardenlet/bootstrap/certificate"
 	"github.com/gardener/gardener/pkg/gardenlet/controller"
 	gardenerhealthz "github.com/gardener/gardener/pkg/healthz"
+	resourcemanagerv1alpha1 "github.com/gardener/gardener/pkg/resourcemanager/apis/config/v1alpha1"
 	"github.com/gardener/gardener/pkg/utils"
 	"github.com/gardener/gardener/pkg/utils/flow"
 	gardenerutils "github.com/gardener/gardener/pkg/utils/gardener"
@@ -83,8 +93,10 @@ import (
 	thirdpartyapiutil "github.com/gardener/gardener/third_party/controller-runtime/pkg/apiutil"
 )
 
-// Name is a const for the name of this component.
-const Name = "gardenlet"
+const (
+	// Name is a const for the name of this component.
+	Name = "gardenlet"
+)
 
 // NewCommand creates a new cobra.Command for running gardenlet.
 func NewCommand() *cobra.Command {
@@ -396,6 +408,10 @@ func (g *garden) Start(ctx context.Context) error {
 	if err := migrateDeprecatedTopologyLabels(ctx, log, g.mgr.GetClient(), g.mgr.GetConfig()); err != nil {
 		return err
 	}
+	log.Info("Creating new secret and managed resource required by dependency-watchdog")
+	if err := g.createNewDWDResources(ctx, g.mgr.GetClient()); err != nil {
+		return err
+	}
 
 	log.Info("Setting up shoot client map")
 	shootClientMap, err := clientmapbuilder.
@@ -446,6 +462,132 @@ func (g *garden) Start(ctx context.Context) error {
 	return nil
 }
 
+// TODO(aaronfern): Remove this code after v1.93 has been released.
+func (g *garden) createNewDWDResources(ctx context.Context, seedClient client.Client) error {
+	namespaceList := &corev1.NamespaceList{}
+	if err := seedClient.List(ctx, namespaceList, client.MatchingLabels(map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleShoot})); err != nil {
+		return err
+	}
+
+	var tasks []flow.TaskFn
+	for _, ns := range namespaceList.Items {
+		if ns.DeletionTimestamp != nil || ns.Status.Phase == corev1.NamespaceTerminating {
+			continue
+		}
+		namespace := ns
+		tasks = append(tasks, func(ctx context.Context) error {
+			dwdOldSecret := &corev1.Secret{}
+			if err := seedClient.Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: dwd.InternalProbeSecretName}, dwdOldSecret); err != nil {
+				// If ns does not contain old DWD secret, do not procees.
+				if apierrors.IsNotFound(err) {
+					return nil
+				}
+				return err
+			}
+
+			// Fetch GRM deployment
+			grmDeploy := &appsv1.Deployment{}
+			if err := seedClient.Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: "gardener-resource-manager"}, grmDeploy); err != nil {
+				if apierrors.IsNotFound(err) {
+					// Do not proceed if GRM deployment is not present
+					return nil
+				}
+				return err
+			}
+
+			// Create a DWDAccess object
+			inClusterServerURL := fmt.Sprintf("%s.%s.svc", v1beta1constants.DeploymentNameKubeAPIServer, namespace.Name)
+			dwdAccess := dwd.NewAccess(seedClient, namespace.Name, nil, dwd.AccessValues{ServerInCluster: inClusterServerURL})
+
+			if err := dwdAccess.DeployMigrate(ctx); err != nil {
+				return err
+			}
+
+			// Delete old DWD secrets
+			if err := kubernetesutils.DeleteObjects(ctx, seedClient,
+				&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: dwd.InternalProbeSecretName, Namespace: namespace.Name}},
+				&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: dwd.ExternalProbeSecretName, Namespace: namespace.Name}},
+			); err != nil {
+				return err
+			}
+
+			// Fetch and update the GRM configmap
+			var grmCMName string
+			var grmCMVolumeIndex int
+			for n, vol := range grmDeploy.Spec.Template.Spec.Volumes {
+				if vol.Name == "config" {
+					grmCMName = vol.ConfigMap.Name
+					grmCMVolumeIndex = n
+					break
+				}
+			}
+			if len(grmCMName) == 0 {
+				return nil
+			}
+
+			grmConfigMap := &corev1.ConfigMap{}
+			if err := seedClient.Get(ctx, types.NamespacedName{Namespace: namespace.Name, Name: grmCMName}, grmConfigMap); err != nil {
+				if apierrors.IsNotFound(err) {
+					return nil
+				}
+				return err
+			}
+
+			cmData := grmConfigMap.Data["config.yaml"]
+			rmConfig := resourcemanagerv1alpha1.ResourceManagerConfiguration{}
+
+			// create codec
+			var codec runtime.Codec
+			configScheme := runtime.NewScheme()
+			utilruntime.Must(resourcemanagerv1alpha1.AddToScheme(configScheme))
+			utilruntime.Must(apiextensionsv1.AddToScheme(configScheme))
+			ser := json.NewSerializerWithOptions(json.DefaultMetaFactory, configScheme, configScheme, json.SerializerOptions{
+				Yaml:   true,
+				Pretty: false,
+				Strict: false,
+			})
+			versions := schema.GroupVersions([]schema.GroupVersion{
+				resourcemanagerv1alpha1.SchemeGroupVersion,
+				apiextensionsv1.SchemeGroupVersion,
+			})
+			codec = serializer.NewCodecFactory(configScheme).CodecForVersions(ser, ser, versions, versions)
+
+			obj, err := runtime.Decode(codec, []byte(cmData))
+			if err != nil {
+				return err
+			}
+			rmConfig = *(obj.(*resourcemanagerv1alpha1.ResourceManagerConfiguration))
+
+			if rmConfig.TargetClientConnection == nil || slices.Contains(rmConfig.TargetClientConnection.Namespaces, corev1.NamespaceNodeLease) {
+				return nil
+			}
+
+			rmConfig.TargetClientConnection.Namespaces = append(rmConfig.TargetClientConnection.Namespaces, corev1.NamespaceNodeLease)
+
+			data, err := runtime.Encode(codec, &rmConfig)
+			if err != nil {
+				return err
+			}
+
+			newGRMConfigMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "gardener-resource-manager-dwd", Namespace: namespace.Name}}
+			newGRMConfigMap.Data = map[string]string{"config.yaml": string(data)}
+			utilruntime.Must(kubernetesutils.MakeUnique(newGRMConfigMap))
+
+			if err = seedClient.Create(ctx, newGRMConfigMap); err != nil {
+				if !apierrors.IsAlreadyExists(err) {
+					return err
+				}
+			}
+
+			patch := client.MergeFrom(grmDeploy.DeepCopy())
+			grmDeploy.Spec.Template.Spec.Volumes[grmCMVolumeIndex].ConfigMap.Name = newGRMConfigMap.Name
+
+			return seedClient.Patch(ctx, grmDeploy, patch)
+		})
+	}
+	return flow.Parallel(tasks...)(ctx)
+}
+
 // TODO(Kostov6): Remove this code after v1.91 has been released.
 func cleanupGRMSecretFinalizers(ctx context.Context, seedClient client.Client, log logr.Logger) error {
 	var (

docs/deployment/feature_gates.md

@@ -33,6 +33,7 @@ The following tables are a summary of the feature gates that you can set on diff
 | APIServerFastRollout                | `true`  | `Beta`  | `1.82` |        |
 | UseGardenerNodeAgent                | `false` | `Alpha` | `1.82` | `1.88` |
 | UseGardenerNodeAgent                | `true`  | `Beta`  | `1.89` |        |
+| DisableAPIServerProxyPort           | `false` | `Alpha` | `1.96` |        |
 
 ## Feature Gates for Graduated or Deprecated Features
 
@@ -193,3 +194,4 @@ A *General Availability* (GA) feature is also referred to as a *stable* feature.
 | ShootForceDeletion                 | `gardener-apiserver`              | Allows forceful deletion of Shoots by annotating them with the `confirmation.gardener.cloud/force-deletion` annotation.                                                                                                                                                                                                                                                            |
 | APIServerFastRollout               | `gardenlet`                       | Enables fast rollouts for Shoot kube-apiservers on the given Seed. When enabled, `maxSurge` for Shoot kube-apiserver deployments is set to 100%.                                                                                                                                                                                                                                   |
 | UseGardenerNodeAgent               | `gardenlet`                       | Enables the `gardener-node-agent` instead of the `cloud-config-downloader` for shoot worker nodes.                                                                                                                                                                                                                                                                                 |
+| DisableAPIServerProxyPort       | `gardenlet`                       | Disables the proxy port (8443) on the istio-ingressgateway Services. It was previously used by the apiserver-proxy to route client traffic on the kubernetes Service to the corresponding API server using the TCP proxy protocol. As soon as a shoot has been reconciled by gardener v1.96+, the apiserver-proxy is reconfigured to use HTTP CONNECT on the tls-tunnel port (8132) instead, i.e., it reuses the reversed VPN path to connect to the correct API server. Operators can choose to remove the legacy apiserver-proxy port as soon as all shoots have switched to the new apiserver-proxy configuration. They might want to do so if they activate the ACL extension, which is vulnerable to proxy protocol headers of untrusted clients on the apiserver-proxy port. |

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/containerd/containerd v1.6.26
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/fluent/fluent-operator/v2 v2.2.0
-	github.com/gardener/dependency-watchdog v1.1.2
+	github.com/gardener/dependency-watchdog v1.2.1
 	github.com/gardener/etcd-druid v0.22.0
 	github.com/gardener/hvpa-controller/api v0.5.0
 	github.com/gardener/machine-controller-manager v0.50.0
@@ -203,3 +203,5 @@ require (
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 )
+
+replace github.com/gardener/etcd-druid => github.com/stackitcloud/etcd-druid v0.22.0-ske-1

go.sum

@@ -369,10 +369,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
-github.com/gardener/dependency-watchdog v1.1.2 h1:id9FAnjL9kiZec+QNVacw4BlxLrI3deMtN4KyxpCcKk=
-github.com/gardener/dependency-watchdog v1.1.2/go.mod h1:giWCBTBkZiY00dv06/DpASzPgc0U+XVF+ZOGkTUewjk=
-github.com/gardener/etcd-druid v0.22.0 h1:DVe+Zjrb93r9vI1uUiCTMHBffIUoMAKhNzFZNC6hsQ8=
-github.com/gardener/etcd-druid v0.22.0/go.mod h1:FROhfVKyWBo4krlPe3R6FIhJRmOmijEWBdEeUP0CJjE=
+github.com/gardener/dependency-watchdog v1.2.1 h1:Q0zqinZNImBuNYfNQGAXkUh5qrfJyrynO5QjUTzO/7w=
+github.com/gardener/dependency-watchdog v1.2.1/go.mod h1:RgU0VmsdBHxRU8IO9VsLxEinz58xEJdEz5hxvMqLKHQ=
 github.com/gardener/hvpa-controller/api v0.5.0 h1:f4F3O7YUrenwh4S3TgPREPiB287JjjUiUL18OqPLyAA=
 github.com/gardener/hvpa-controller/api v0.5.0/go.mod h1:QQl3ELkCaki+8RhXl0FZMfvnm0WCGwGJlGmrxJj6lvM=
 github.com/gardener/machine-controller-manager v0.50.0 h1:3dcQjzueFU1TGgprV00adjb3OCR99myTBx8DQGxywks=
@@ -881,6 +879,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc=
 github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg=
+github.com/stackitcloud/etcd-druid v0.22.0-ske-1 h1:H59mbOygVBJdIM0L9Byb5SrpasMdBdQGDmezlCsyvVA=
+github.com/stackitcloud/etcd-druid v0.22.0-ske-1/go.mod h1:FROhfVKyWBo4krlPe3R6FIhJRmOmijEWBdEeUP0CJjE=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
 github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=

imagevector/images.yaml

@@ -58,7 +58,7 @@ images:
 - name: dependency-watchdog
   sourceRepository: github.com/gardener/dependency-watchdog
   repository: europe-docker.pkg.dev/gardener-project/releases/gardener/dependency-watchdog
-  tag: "v1.1.2"
+  tag: "v1.2.3"
 - name: nginx-ingress-controller
   sourceRepository: github.com/kubernetes/ingress-nginx
   repository: registry.k8s.io/ingress-nginx/controller-chroot
@@ -664,8 +664,9 @@ images:
 # External Authorization Server for the Istio Endpoint of Reversed VPN
 - name: ext-authz-server
   sourceRepository: github.com/gardener/ext-authz-server
-  repository: europe-docker.pkg.dev/gardener-project/releases/gardener/ext-authz-server
-  tag: "0.9.0"
+  # built from https://github.com/stackitcloud/ext-authz-server/tree/hackathon-apiserver-proxy
+  repository: ghcr.io/stackitcloud/ext-authz-server
+  tag: "0.10.0-dev-34b23c6"
 
 # API Server SNI
 - name: apiserver-proxy

pkg/component/apiserverproxy/apiserver_proxy.go

@@ -37,6 +37,7 @@ import (
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
 	"github.com/gardener/gardener/pkg/client/kubernetes"
 	"github.com/gardener/gardener/pkg/component"
+	"github.com/gardener/gardener/pkg/component/vpnseedserver"
 	"github.com/gardener/gardener/pkg/resourcemanager/controller/garbagecollector/references"
 	"github.com/gardener/gardener/pkg/utils"
 	kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes"
@@ -53,7 +54,8 @@ const (
 	webhookExpressionsKey = "apiserver-proxy.networking.gardener.cloud/inject"
 
 	adminPort           = 16910
-	proxySeedServerPort = 8443
+	proxySeedServerPort = vpnseedserver.GatewayPort
+	portNameMetrics     = "metrics"
 
 	volumeNameConfig   = "proxy-config"
 	volumeNameAdminUDS = "admin-uds"
@@ -163,6 +165,7 @@ func (a *apiserverProxy) computeResourcesData() (map[string][]byte, error) {
 		"adminPort":           adminPort,
 		"proxySeedServerHost": a.values.ProxySeedServerHost,
 		"proxySeedServerPort": proxySeedServerPort,
+		"namespace":           a.namespace,
 	}); err != nil {
 		return nil, err
 	}

pkg/component/apiserverproxy/apiserver_proxy_test.go

@@ -472,6 +472,12 @@ data:
               "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
               stat_prefix: kube_apiserver
               cluster: kube_apiserver
+              tunneling_config:
+                hostname: "api.internal.local.:443"
+                headers_to_add:
+                - header:
+                    key: Reversed-VPN
+                    value: "outbound|443||kube-apiserver.some-namespace.svc.cluster.local"
               access_log:
               - name: envoy.access_loggers.stdout
                 typed_config:
@@ -544,17 +550,7 @@ data:
                 address:
                   socket_address:
                     address: api.internal.local.
-                    port_value: 8443
-        transport_socket:
-          name: envoy.transport_sockets.upstream_proxy_protocol
-          typed_config:
-            "@type": type.googleapis.com/envoy.extensions.transport_sockets.proxy_protocol.v3.ProxyProtocolUpstreamTransport
-            config:
-              version: V2
-            transport_socket:
-              name: envoy.transport_sockets.raw_buffer
-              typed_config:
-                "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer
+                    port_value: 8132
         upstream_connection_options:
           tcp_keepalive:
             keepalive_time: 7200

pkg/component/apiserverproxy/templates/envoy.yaml.tpl

@@ -51,6 +51,12 @@ static_resources:
           "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           stat_prefix: kube_apiserver
           cluster: kube_apiserver
+          tunneling_config:
+            hostname: "{{ .proxySeedServerHost }}:443"
+            headers_to_add:
+            - header:
+                key: Reversed-VPN
+                value: "outbound|443||kube-apiserver.{{ .namespace }}.svc.cluster.local"
           access_log:
           - name: envoy.access_loggers.stdout
             typed_config:
@@ -124,16 +130,6 @@ static_resources:
               socket_address:
                 address: {{ .proxySeedServerHost }}
                 port_value: {{ .proxySeedServerPort }}
-    transport_socket:
-      name: envoy.transport_sockets.upstream_proxy_protocol
-      typed_config:
-        "@type": type.googleapis.com/envoy.extensions.transport_sockets.proxy_protocol.v3.ProxyProtocolUpstreamTransport
-        config:
-          version: V2
-        transport_socket:
-          name: envoy.transport_sockets.raw_buffer
-          typed_config:
-            "@type": type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer
     upstream_connection_options:
       tcp_keepalive:
         keepalive_time: 7200