From 3ffd26693298a34c9665c167173f3bbc001a4b98 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Wed, 13 Nov 2024 12:54:12 +0100 Subject: [PATCH] Move all kubelet config logic into the write method This allows for dissolving the config struct and make the assignments directly to the generated kubelet configuration struct. Removes unnecessary indirection and concentrates more of the decision making into the appropriate method. Defer OS-specific decisions about the path to resolv.conf to the specific function, so that its return values are meaningful even if it's not called on Linux. Set the cgroups settings and do systemd detection only on Linux, as these are Linux-specific and don't exist on other operating systems. Signed-off-by: Tom Wieczorek --- pkg/component/worker/kubelet.go | 98 ++++++++++++++------------------- 1 file changed, 42 insertions(+), 56 deletions(-) diff --git a/pkg/component/worker/kubelet.go b/pkg/component/worker/kubelet.go index 431732189de3..0d27ce2374b9 100644 --- a/pkg/component/worker/kubelet.go +++ b/pkg/component/worker/kubelet.go @@ -69,16 +69,6 @@ type Kubelet struct { var _ manager.Component = (*Kubelet)(nil) -type kubeletConfig struct { - ClientCAFile string - VolumePluginDir string - KubeReservedCgroup string - KubeletCgroups string - CgroupsPerQOS bool - ResolvConf string - StaticPodURL string -} - // Init extracts the needed binaries func (k *Kubelet) Init(_ context.Context) error { cmds := []string{"kubelet", "xtables-legacy-multi", "xtables-nft-multi"} @@ -156,21 +146,6 @@ func lookupHostname(ctx context.Context, hostname string) (ipv4 net.IP, ipv6 net func (k *Kubelet) Start(ctx context.Context) error { cmd := "kubelet" - var staticPodURL string - if k.StaticPods != nil { - var err error - if staticPodURL, err = k.StaticPods.ManifestURL(); err != nil { - return err - } - } - - kubeletConfigData := kubeletConfig{ - ClientCAFile: filepath.Join(k.K0sVars.CertRootDir, "ca.crt"), - VolumePluginDir: k.K0sVars.KubeletVolumePluginDir, - KubeReservedCgroup: "system.slice", - KubeletCgroups: "/system.slice/containerd.service", - StaticPodURL: staticPodURL, - } if runtime.GOOS == "windows" { cmd = "kubelet.exe" } @@ -211,15 +186,10 @@ func (k *Kubelet) Start(ctx context.Context) error { } if runtime.GOOS == "windows" { - kubeletConfigData.CgroupsPerQOS = false - kubeletConfigData.ResolvConf = "" args["--enforce-node-allocatable"] = "" args["--hostname-override"] = nodename args["--hairpin-mode"] = "promiscuous-bridge" args["--cert-dir"] = "C:\\var\\lib\\k0s\\kubelet_certs" - } else { - kubeletConfigData.CgroupsPerQOS = true - kubeletConfigData.ResolvConf = determineKubeletResolvConfPath() } if k.CRISocket == "" && runtime.GOOS != "windows" { @@ -247,7 +217,7 @@ func (k *Kubelet) Start(ctx context.Context) error { Args: args.ToArgs(), } - if err := k.writeKubeletConfig(kubeletConfigData, kubeletConfigPath); err != nil { + if err := k.writeKubeletConfig(kubeletConfigPath); err != nil { return err } @@ -260,20 +230,25 @@ func (k *Kubelet) Stop() error { return nil } -func (k *Kubelet) writeKubeletConfig(kubeletConfigData kubeletConfig, path string) error { - config := k.Configuration.DeepCopy() - config.Authentication.X509.ClientCAFile = kubeletConfigData.ClientCAFile // filepath.Join(k.K0sVars.CertRootDir, "ca.crt") - config.VolumePluginDir = kubeletConfigData.VolumePluginDir // k.K0sVars.KubeletVolumePluginDir - config.KubeReservedCgroup = kubeletConfigData.KubeReservedCgroup - config.KubeletCgroups = kubeletConfigData.KubeletCgroups - config.ResolverConfig = ptr.To(kubeletConfigData.ResolvConf) - config.CgroupsPerQOS = ptr.To(kubeletConfigData.CgroupsPerQOS) - config.StaticPodURL = kubeletConfigData.StaticPodURL +func (k *Kubelet) writeKubeletConfig(path string) error { + var staticPodURL string + if k.StaticPods != nil { + var err error + if staticPodURL, err = k.StaticPods.ManifestURL(); err != nil { + return err + } + } containerRuntimeEndpoint, err := GetContainerRuntimeEndpoint(k.CRISocket, k.K0sVars.RunDir) if err != nil { return err } + + config := k.Configuration.DeepCopy() + config.Authentication.X509.ClientCAFile = filepath.Join(k.K0sVars.CertRootDir, "ca.crt") + config.VolumePluginDir = k.K0sVars.KubeletVolumePluginDir + config.ResolverConfig = determineKubeletResolvConfPath() + config.StaticPodURL = staticPodURL config.ContainerRuntimeEndpoint = containerRuntimeEndpoint.String() if len(k.Taints) > 0 { @@ -288,6 +263,13 @@ func (k *Kubelet) writeKubeletConfig(kubeletConfigData kubeletConfig, path strin config.RegisterWithTaints = taints } + // cgroup related things (Linux only) + if runtime.GOOS == "linux" { + config.KubeReservedCgroup = "system.slice" + config.KubeletCgroups = "/system.slice/containerd.service" + config.CgroupsPerQOS = ptr.To(true) + } + configBytes, err := yaml.Marshal(config) if err != nil { return fmt.Errorf("can't marshal kubelet config: %w", err) @@ -354,27 +336,31 @@ func validateTaintEffect(effect corev1.TaintEffect) error { // determineKubeletResolvConfPath returns the path to the resolv.conf file that // the kubelet should use. -func determineKubeletResolvConfPath() string { +func determineKubeletResolvConfPath() *string { path := "/etc/resolv.conf" - // https://www.freedesktop.org/software/systemd/man/systemd-resolved.service.html#/etc/resolv.conf - // If it's likely that resolv.conf is pointing to a systemd-resolved - // nameserver, that nameserver won't be reachable from within containers. - // Try to use the alternative resolv.conf path used by systemd-resolved instead. - detected, err := hasSystemdResolvedNameserver(path) - if err != nil { - logrus.WithError(err).Infof("Error while trying to detect the presence of systemd-resolved, using resolv.conf: %s", path) - return path - } - - if detected { - alternatePath := "/run/systemd/resolve/resolv.conf" - logrus.Infof("The file %s looks like it's managed by systemd-resolved, using resolv.conf: %s", path, alternatePath) - return alternatePath + switch runtime.GOOS { + case "windows": + return nil + + case "linux": + // https://www.freedesktop.org/software/systemd/man/systemd-resolved.service.html#/etc/resolv.conf + // If it's likely that resolv.conf is pointing to a systemd-resolved + // nameserver, that nameserver won't be reachable from within + // containers. Try to use the alternative resolv.conf path used by + // systemd-resolved instead. + detected, err := hasSystemdResolvedNameserver(path) + if err != nil { + logrus.WithError(err).Info("Failed to detect the presence of systemd-resolved") + } else if detected { + systemdPath := "/run/systemd/resolve/resolv.conf" + logrus.Infof("The file %s looks like it's managed by systemd-resolved, using resolv.conf: %s", path, systemdPath) + return &systemdPath + } } logrus.Infof("Using resolv.conf: %s", path) - return path + return &path } // hasSystemdResolvedNameserver parses the given resolv.conf file and checks if