From a8dc9165eadc2abc8d41315fcc686664bfe9bca3 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 11 Oct 2023 22:29:54 +0000 Subject: [PATCH 1/7] Bump kubernetes to v1.26.9-bd1-rke2r1 Signed-off-by: Brad Davidson --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d9379a7100..9bdbece366 100644 --- a/Dockerfile +++ b/Dockerfile @@ -138,7 +138,7 @@ RUN rm -vf /charts/*.sh /charts/*.md # This image includes any host level programs that we might need. All binaries # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation. # This means bin/foo/bar will become bin/bar when rke2 installs this to the host -FROM rancher/hardened-kubernetes:v1.26.9-rke2r1-build20230913 AS kubernetes +FROM brandond/hardened-kubernetes:v1.26.9-bd1-rke2r1-build20231011 AS kubernetes FROM rancher/hardened-containerd:v1.7.7-k3s1-build20231010 AS containerd FROM rancher/hardened-crictl:v1.26.1-build20230406 AS crictl FROM rancher/hardened-runc:v1.1.8-build20230802 AS runc From 03df00430069394e7a0d2c05b60bf974b52a391c Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 13 Oct 2023 06:53:18 +0000 Subject: [PATCH 2/7] Bump kubernetes to v1.26.9-bd1-rke2r1 Signed-off-by: Brad Davidson --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9bdbece366..fa5ef77b54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -138,7 +138,7 @@ RUN rm -vf /charts/*.sh /charts/*.md # This image includes any host level programs that we might need. All binaries # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation. # This means bin/foo/bar will become bin/bar when rke2 installs this to the host -FROM brandond/hardened-kubernetes:v1.26.9-bd1-rke2r1-build20231011 AS kubernetes +FROM brandond/hardened-kubernetes:v1.26.9-bd2-rke2r1-build20231013 AS kubernetes FROM rancher/hardened-containerd:v1.7.7-k3s1-build20231010 AS containerd FROM rancher/hardened-crictl:v1.26.1-build20230406 AS crictl FROM rancher/hardened-runc:v1.1.8-build20230802 AS runc From 5171cfe9a7f68088b02347bc58d6b9ef3c392065 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 17 Oct 2023 22:09:21 +0000 Subject: [PATCH 3/7] Bump kubernetes to v1.26.9-bd3-rke2r1-build20231017 Signed-off-by: Brad Davidson --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fa5ef77b54..761cb50511 100644 --- a/Dockerfile +++ b/Dockerfile @@ -138,7 +138,7 @@ RUN rm -vf /charts/*.sh /charts/*.md # This image includes any host level programs that we might need. All binaries # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation. # This means bin/foo/bar will become bin/bar when rke2 installs this to the host -FROM brandond/hardened-kubernetes:v1.26.9-bd2-rke2r1-build20231013 AS kubernetes +FROM brandond/hardened-kubernetes:v1.26.9-bd3-rke2r1-build20231017 AS kubernetes FROM rancher/hardened-containerd:v1.7.7-k3s1-build20231010 AS containerd FROM rancher/hardened-crictl:v1.26.1-build20230406 AS crictl FROM rancher/hardened-runc:v1.1.8-build20230802 AS runc From 0f2f8ac3735a8944eda9d70d3072b6acae76e784 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 18 Oct 2023 02:22:10 +0300 Subject: [PATCH 4/7] remove /var/lib/kubelet in killall script Signed-off-by: galal-hussein --- bundle/bin/rke2-killall.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/bin/rke2-killall.sh b/bundle/bin/rke2-killall.sh index 3f3e3bb3f8..ef53353350 100755 --- a/bundle/bin/rke2-killall.sh +++ b/bundle/bin/rke2-killall.sh @@ -90,6 +90,7 @@ if [ -d /sys/class/net/nodelocaldns ]; then fi rm -rf /var/lib/cni/ /var/log/pods/ /var/log/containers +rm -rf /var/lib/kubelet || true # Delete iptables created by CNI plugins or Kubernetes (kube-proxy) iptables-save | grep -v KUBE- | grep -v CNI- | grep -v cali- | grep -v cali: | grep -v CILIUM_ | grep -v flannel | iptables-restore From 8f79ac7d540e0b56cb454970b0f5d0be8e6d1ca0 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 18 Oct 2023 19:18:26 +0300 Subject: [PATCH 5/7] remove containerd files in killall script Signed-off-by: galal-hussein --- bundle/bin/rke2-killall.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/bin/rke2-killall.sh b/bundle/bin/rke2-killall.sh index ef53353350..2484e2185b 100755 --- a/bundle/bin/rke2-killall.sh +++ b/bundle/bin/rke2-killall.sh @@ -64,6 +64,9 @@ do_unmount_and_remove '/var/lib/rancher/rke2' do_unmount_and_remove '/var/lib/kubelet/pods' do_unmount_and_remove '/run/netns/cni-' +# Delete old containerd files +rm -rf /var/lib/rancher/rke2/agent/containerd/ || true + # Delete network interface(s) that match 'master cni0' ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do iface=${iface%%@*} From 8c42d0b8d43db3deb4a1feb4e6b81473d320edd1 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Wed, 18 Oct 2023 20:49:19 +0300 Subject: [PATCH 6/7] Add readiness probe to kube-controller-manager and cloud-controller-manager Signed-off-by: galal-hussein --- pkg/podexecutor/staticpod.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index 080643059d..4144db0fe6 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -445,6 +445,8 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s CISMode: s.CISMode, HealthPort: 10257, HealthProto: "HTTPS", + ReadyPort: 10257, + ReadyProto: "HTTPS", CPURequest: s.ControlPlaneResources.KubeControllerManagerCPURequest, CPULimit: s.ControlPlaneResources.KubeControllerManagerCPULimit, MemoryRequest: s.ControlPlaneResources.KubeControllerManagerMemoryRequest, @@ -457,7 +459,7 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s }) } -// CloudControllerManager starts the cloud-controller-manager static pod, once the cloud controller manager RBAC +// CloudControllerManager starts the -manager static pod, once the cloud controller manager RBAC // (and subsequently, the api server) is available. func (s *StaticPodConfig) CloudControllerManager(_ context.Context, ccmRBACReady <-chan struct{}, args []string) error { image, err := s.Resolver.GetReference(images.CloudControllerManager) @@ -476,6 +478,8 @@ func (s *StaticPodConfig) CloudControllerManager(_ context.Context, ccmRBACReady CISMode: s.CISMode, HealthPort: 10258, HealthProto: "HTTPS", + ReadyPort: 10257, + ReadyProto: "HTTPS", CPURequest: s.ControlPlaneResources.CloudControllerManagerCPURequest, CPULimit: s.ControlPlaneResources.CloudControllerManagerCPULimit, MemoryRequest: s.ControlPlaneResources.CloudControllerManagerMemoryRequest, From 2f87a7fdd73201f9e11aa054034f192277aad7b1 Mon Sep 17 00:00:00 2001 From: galal-hussein Date: Thu, 19 Oct 2023 02:38:45 +0300 Subject: [PATCH 7/7] remove pod-manifests dir in killall script Signed-off-by: galal-hussein --- bundle/bin/rke2-killall.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/bin/rke2-killall.sh b/bundle/bin/rke2-killall.sh index 2484e2185b..a33eaf881d 100755 --- a/bundle/bin/rke2-killall.sh +++ b/bundle/bin/rke2-killall.sh @@ -64,8 +64,9 @@ do_unmount_and_remove '/var/lib/rancher/rke2' do_unmount_and_remove '/var/lib/kubelet/pods' do_unmount_and_remove '/run/netns/cni-' -# Delete old containerd files +# Delete old containerd files and pod-manfiests rm -rf /var/lib/rancher/rke2/agent/containerd/ || true +rm -rf /var/lib/rancher/rke2/agent/pod-manifests || true # Delete network interface(s) that match 'master cni0' ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do