From d66aca51cbb8d54229cb6a4d2cc44a74f6d0458b Mon Sep 17 00:00:00 2001 From: Vitor Savian Date: Tue, 3 Dec 2024 23:42:56 +0000 Subject: [PATCH 1/4] Add runtime classes hook and runtimes chart Signed-off-by: Vitor Savian --- charts/chart_versions.yaml | 3 ++ pkg/rke2/rc.go | 85 ++++++++++++++++++++++++++++++++++++++ pkg/rke2/rke2.go | 1 + 3 files changed, 89 insertions(+) create mode 100644 pkg/rke2/rc.go diff --git a/charts/chart_versions.yaml b/charts/chart_versions.yaml index 24d7ebb2ca..b6de23e157 100644 --- a/charts/chart_versions.yaml +++ b/charts/chart_versions.yaml @@ -53,3 +53,6 @@ charts: - version: 1.9.001 filename: /charts/rke2-snapshot-validation-webhook.yaml bootstrap: false + - version: 0.1.000 + filename: /charts/rke2-runtimeclasses.yaml + bootstrap: false diff --git a/pkg/rke2/rc.go b/pkg/rke2/rc.go new file mode 100644 index 0000000000..e0265670e9 --- /dev/null +++ b/pkg/rke2/rc.go @@ -0,0 +1,85 @@ +package rke2 + +import ( + "context" + "sync" + + "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const runtimeClassesChart = "rke2-runtimeclasses" + +var runtimes = map[string]bool{ + "nvidia": true, + "nvidia-experimental": true, + "crun": true, +} + +func setRuntimes() cmds.StartupHook { + return func(ctx context.Context, wg *sync.WaitGroup, args cmds.StartupHookArgs) error { + go func() { + defer wg.Done() + <-args.APIServerReady + logrus.Info("Setting runtimes") + + config, err := clientcmd.BuildConfigFromFlags("", args.KubeConfigSupervisor) + if err != nil { + logrus.Fatalf("runtimes: new k8s restConfig: %v", err) + } + + client, err := kubernetes.NewForConfig(config) + if err != nil { + logrus.Fatalf("runtimes: new k8s client: %v", err) + } + + rcClient := client.NodeV1().RuntimeClasses() + + classes, err := rcClient.List(context.Background(), metav1.ListOptions{}) + if err != nil { + logrus.Fatalf("runtimes: failed to get runtime classes") + } + + for _, c := range classes.Items { + + // verify if the runtime class is the runtime class supported by rke2 + if _, ok := runtimes[c.Name]; !ok { + continue + } + + if c.Labels == nil { + labels := make(map[string]string) + c.SetLabels(labels) + } + + if managedBy, ok := c.Labels["app.kubernetes.io/managed-by"]; !ok || managedBy != "Helm" { + c.Labels["app.kubernetes.io/managed-by"] = "Helm" + } + + if c.Annotations == nil { + annotations := make(map[string]string) + c.SetAnnotations(annotations) + } + + if releaseName, ok := c.Annotations["meta.helm.sh/release-name"]; !ok || releaseName != runtimeClassesChart { + c.Annotations["meta.helm.sh/release-name"] = runtimeClassesChart + } + + if namespace, ok := c.Annotations["meta.helm.sh/release-namespace"]; !ok || namespace != "kube-system" { + c.Annotations["meta.helm.sh/release-namespace"] = "kube-system" + } + + _, err = rcClient.Update(context.Background(), &c, metav1.UpdateOptions{}) + if err != nil { + logrus.Fatalf("runtimes: failed to update runtime classes") + } + + } + }() + + return nil + } +} diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 1d0dacce46..8695044aae 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -138,6 +138,7 @@ func Server(clx *cli.Context, cfg Config) error { restrictServiceAccounts(cisMode, defaultNamespaces), setKubeProxyDisabled(), cleanupStaticPodsOnSelfDelete(dataDir), + setRuntimes(), ) var leaderControllers rawServer.CustomControllers From 6a6325de1c61aa75672f788b8986b35a09f28a83 Mon Sep 17 00:00:00 2001 From: Vitor Savian Date: Wed, 4 Dec 2024 00:58:56 +0000 Subject: [PATCH 2/4] Addressing comments Signed-off-by: Vitor Savian --- pkg/rke2/rc.go | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/pkg/rke2/rc.go b/pkg/rke2/rc.go index e0265670e9..4ec350b0ee 100644 --- a/pkg/rke2/rc.go +++ b/pkg/rke2/rc.go @@ -5,13 +5,19 @@ import ( "sync" "github.com/k3s-io/k3s/pkg/cli/cmds" + "github.com/k3s-io/k3s/pkg/util" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" ) -const runtimeClassesChart = "rke2-runtimeclasses" +const ( + runtimeClassesChart = "rke2-runtimeclasses" + namespace = "kube-system" + helm = "Helm" + helmReleaseName = "meta.helm.sh/release-name" + helmManageBy = "app.kubernetes.io/managed-by" + helmReleaseNamespace = "meta.helm.sh/release-namespace" +) var runtimes = map[string]bool{ "nvidia": true, @@ -26,12 +32,7 @@ func setRuntimes() cmds.StartupHook { <-args.APIServerReady logrus.Info("Setting runtimes") - config, err := clientcmd.BuildConfigFromFlags("", args.KubeConfigSupervisor) - if err != nil { - logrus.Fatalf("runtimes: new k8s restConfig: %v", err) - } - - client, err := kubernetes.NewForConfig(config) + client, err := util.GetClientSet(args.KubeConfigSupervisor) if err != nil { logrus.Fatalf("runtimes: new k8s client: %v", err) } @@ -51,25 +52,23 @@ func setRuntimes() cmds.StartupHook { } if c.Labels == nil { - labels := make(map[string]string) - c.SetLabels(labels) + c.Labels = map[string]string{} } - if managedBy, ok := c.Labels["app.kubernetes.io/managed-by"]; !ok || managedBy != "Helm" { - c.Labels["app.kubernetes.io/managed-by"] = "Helm" + if managedBy, ok := c.Labels[helmManageBy]; !ok || managedBy != helm { + c.Labels[helmManageBy] = helm } if c.Annotations == nil { - annotations := make(map[string]string) - c.SetAnnotations(annotations) + c.Annotations = map[string]string{} } - if releaseName, ok := c.Annotations["meta.helm.sh/release-name"]; !ok || releaseName != runtimeClassesChart { - c.Annotations["meta.helm.sh/release-name"] = runtimeClassesChart + if releaseName, ok := c.Annotations[helmReleaseName]; !ok || releaseName != runtimeClassesChart { + c.Annotations[helmReleaseName] = runtimeClassesChart } - if namespace, ok := c.Annotations["meta.helm.sh/release-namespace"]; !ok || namespace != "kube-system" { - c.Annotations["meta.helm.sh/release-namespace"] = "kube-system" + if ns, ok := c.Annotations[helmReleaseNamespace]; !ok || ns != namespace { + c.Annotations[helmReleaseNamespace] = namespace } _, err = rcClient.Update(context.Background(), &c, metav1.UpdateOptions{}) From 1b51e4c49ce233b4f3c76269cb1299e41e0b2426 Mon Sep 17 00:00:00 2001 From: Vitor Savian Date: Wed, 4 Dec 2024 02:06:02 +0000 Subject: [PATCH 3/4] Change const name to the same as helm upstream Signed-off-by: Vitor Savian --- pkg/rke2/rc.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/rke2/rc.go b/pkg/rke2/rc.go index 4ec350b0ee..5b24b5018f 100644 --- a/pkg/rke2/rc.go +++ b/pkg/rke2/rc.go @@ -11,12 +11,14 @@ import ( ) const ( - runtimeClassesChart = "rke2-runtimeclasses" - namespace = "kube-system" - helm = "Helm" - helmReleaseName = "meta.helm.sh/release-name" - helmManageBy = "app.kubernetes.io/managed-by" - helmReleaseNamespace = "meta.helm.sh/release-namespace" + runtimeClassesChart = "rke2-runtimeclasses" + defaultNamespace = "kube-system" + + // Values from upstream, see reference at -> https://github.com/helm/helm/blob/v3.16.3/pkg/action/validate.go#L34-L37 + appManagedByLabel = "app.kubernetes.io/managed-by" + appManagedByHelm = "Helm" + helmReleaseNameAnnotation = "meta.helm.sh/release-name" + helmReleaseNamespaceAnnotation = "meta.helm.sh/release-namespace" ) var runtimes = map[string]bool{ @@ -55,20 +57,20 @@ func setRuntimes() cmds.StartupHook { c.Labels = map[string]string{} } - if managedBy, ok := c.Labels[helmManageBy]; !ok || managedBy != helm { - c.Labels[helmManageBy] = helm + if managedBy, ok := c.Labels[appManagedByLabel]; !ok || managedBy != appManagedByHelm { + c.Labels[appManagedByLabel] = appManagedByHelm } if c.Annotations == nil { c.Annotations = map[string]string{} } - if releaseName, ok := c.Annotations[helmReleaseName]; !ok || releaseName != runtimeClassesChart { - c.Annotations[helmReleaseName] = runtimeClassesChart + if releaseName, ok := c.Annotations[helmReleaseNameAnnotation]; !ok || releaseName != runtimeClassesChart { + c.Annotations[helmReleaseNameAnnotation] = runtimeClassesChart } - if ns, ok := c.Annotations[helmReleaseNamespace]; !ok || ns != namespace { - c.Annotations[helmReleaseNamespace] = namespace + if namespace, ok := c.Annotations[helmReleaseNamespaceAnnotation]; !ok || namespace != defaultNamespace { + c.Annotations[helmReleaseNamespaceAnnotation] = defaultNamespace } _, err = rcClient.Update(context.Background(), &c, metav1.UpdateOptions{}) From 116ecff3e4c777124b33815fa5f97608ec3f4ff0 Mon Sep 17 00:00:00 2001 From: Vitor Savian Date: Wed, 4 Dec 2024 17:42:07 +0000 Subject: [PATCH 4/4] Delete namespace const and use upstream namespace system Signed-off-by: Vitor Savian --- pkg/rke2/rc.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/rke2/rc.go b/pkg/rke2/rc.go index 5b24b5018f..4266d75744 100644 --- a/pkg/rke2/rc.go +++ b/pkg/rke2/rc.go @@ -12,7 +12,6 @@ import ( const ( runtimeClassesChart = "rke2-runtimeclasses" - defaultNamespace = "kube-system" // Values from upstream, see reference at -> https://github.com/helm/helm/blob/v3.16.3/pkg/action/validate.go#L34-L37 appManagedByLabel = "app.kubernetes.io/managed-by" @@ -69,8 +68,8 @@ func setRuntimes() cmds.StartupHook { c.Annotations[helmReleaseNameAnnotation] = runtimeClassesChart } - if namespace, ok := c.Annotations[helmReleaseNamespaceAnnotation]; !ok || namespace != defaultNamespace { - c.Annotations[helmReleaseNamespaceAnnotation] = defaultNamespace + if namespace, ok := c.Annotations[helmReleaseNamespaceAnnotation]; !ok || namespace != metav1.NamespaceSystem { + c.Annotations[helmReleaseNamespaceAnnotation] = metav1.NamespaceSystem } _, err = rcClient.Update(context.Background(), &c, metav1.UpdateOptions{})