From 6f1864434ed1f9775107be3e9a0a018cd8a4e715 Mon Sep 17 00:00:00 2001 From: jdambly Date: Wed, 2 Oct 2024 15:02:54 -0700 Subject: [PATCH] initial commit for updated device plugin chart --- sriov/templates/NOTES.txt | 31 --- sriov/templates/_helpers.tpl | 11 - .../configMap_sriov_device_plugin.yaml | 4 +- sriov/templates/daemonSet_sriov_cni.yaml | 30 ++- .../daemonSet_sriov_device_plugin.yaml | 28 ++- sriov/templates/net_attach_def_dpdk.yaml | 26 --- sriov/templates/net_attach_def_netdev.yaml | 26 --- sriov/values.yaml | 201 +++++++----------- 8 files changed, 116 insertions(+), 241 deletions(-) delete mode 100644 sriov/templates/net_attach_def_dpdk.yaml delete mode 100644 sriov/templates/net_attach_def_netdev.yaml diff --git a/sriov/templates/NOTES.txt b/sriov/templates/NOTES.txt index 01b5768..9069691 100644 --- a/sriov/templates/NOTES.txt +++ b/sriov/templates/NOTES.txt @@ -1,37 +1,6 @@ -====== -{{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "16" ) }} -1. The following components have been deployed as part of this helm chart: -{{- if .Values.manifests.serviceAccount }} -Service Account: {{ .Values.serviceAccount.name }} -{{- end }} -{{- if .Values.manifests.configMap_sriov_device_plugin }} -Config Map: {{ .Release.Name }}-{{ .Chart.Name }}-{{ .Chart.Version }}-config -{{- end }} -{{- if .Values.manifests.daemonSet_sriov_device_plugin }} -Daemon Set: {{ .Release.Name }}-{{ .Chart.Name }}-device-plugin-ds -{{- end }} -{{- if .Values.manifests.daemonSet_sriov_cni }} -Daemon Set: {{ .Release.Name }}-{{ .Chart.Name }}-cni-ds -{{- end }} -{{- if .Values.manifests.net_attach_def_netdev }} -Network Attachment Definition for NetDevice: {{ .Values.config.netAttachDef.netdeviceName }} -{{- end }} -{{- if .Values.manifests.net_attach_def_dpdk }} -Network Attachment Definition for DPDK: {{ .Values.config.netAttachDef.dpdkName }} -{{- end }} -{{- if .Values.manifests.test_netdevice }} -Example pod with nedevice interface: {{ .Release.Name }}-test-netdevice -{{- end }} -{{- if .Values.manifests.test_dpdk }} -Example pod with dpdk interface: {{ .Release.Name }}-test-dpdk -{{- end }} - You can now create more Network Attachment Definitions or test SRIOV pods using Network Attachment Defintions as mentioned above. --------- 2. To uninstall helm chart use the command: helm delete {{ .Release.Name }} --------- -{{- else }} -To run these charts, please use K8s ver >= v1.16 -{{- end }} diff --git a/sriov/templates/_helpers.tpl b/sriov/templates/_helpers.tpl index 0e1e040..c46bcdb 100644 --- a/sriov/templates/_helpers.tpl +++ b/sriov/templates/_helpers.tpl @@ -28,14 +28,3 @@ app: {{ .Chart.Name }}-dp name: {{ .Values.serviceAccount.name }} {{- end }} -{{/* Generate SRIOV tolerations */}} -{{- define "sriov.tolerations" }} -- key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule -{{- if index .Values "tolerations" }} -{{- if gt (len .Values.tolerations) 0 }} -{{ toYaml .Values.tolerations }} -{{- end }} -{{- end }} -{{- end }} diff --git a/sriov/templates/configMap_sriov_device_plugin.yaml b/sriov/templates/configMap_sriov_device_plugin.yaml index 5e435c0..fbeb4a3 100644 --- a/sriov/templates/configMap_sriov_device_plugin.yaml +++ b/sriov/templates/configMap_sriov_device_plugin.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "16" ) }} -{{- if .Values.manifests.configMap_sriov_device_plugin }} apiVersion: v1 kind: ConfigMap metadata: @@ -20,6 +19,5 @@ metadata: namespace: kube-system data: dp-conf.json: |- -{{ toJson .Values.config.sriov_device_plugin | indent 4 }} -{{- end }} + {{ toJson .Values.config.sriov_device_plugin | indent 4 }} {{- end }} diff --git a/sriov/templates/daemonSet_sriov_cni.yaml b/sriov/templates/daemonSet_sriov_cni.yaml index 4495c00..8d83c7e 100644 --- a/sriov/templates/daemonSet_sriov_cni.yaml +++ b/sriov/templates/daemonSet_sriov_cni.yaml @@ -12,18 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "16" ) }} -{{- if .Values.manifests.daemonSet_sriov_cni }} apiVersion: apps/v1 kind: DaemonSet metadata: name: {{ .Release.Name }}-{{ .Chart.Name }}-cni-ds namespace: kube-system labels: -{{- include "sriov-cni.labels" . | indent 4 }} + {{- include "sriov-cni.labels" . | nindent 4 }} spec: selector: matchLabels: -{{- include "sriov-cni.labels" . | indent 6 }} + {{- include "sriov-cni.labels" . | nindent 6 }} updateStrategy: type: RollingUpdate rollingUpdate: @@ -31,22 +30,34 @@ spec: template: metadata: labels: -{{- include "sriov-cni.labels" . | indent 8 }} + {{- include "sriov-cni.labels" . | nindent 8 }} spec: hostNetwork: true - nodeSelector: {{- toYaml .Values.labels.nodeSelector | nindent 8 }} + priorityClassName: system-node-critical + {{- with .Values.nodeSelector}} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} tolerations: -{{- include "sriov.tolerations" . | indent 6 }} + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ .Values.serviceAccount.name }} containers: - name: kube-{{ .Chart.Name }}-cni image: {{ .Values.images.sriovCni.repository }}:{{ .Values.images.sriovCni.tag }} imagePullPolicy: {{ .Values.images.pullPolicy }} + {{- with .Values.securityContext}} securityContext: - privileged: {{ .Values.securityContext.privileged }} - {{- if .Values.pod.resources.enabled }} - resources: {{- toYaml .Values.pod.resources.sriov_cni | nindent 10 }} + {{- toYaml . | nindent 10 }} {{- end }} + resources: + requests: + memory: "50Mi" + cpu: "100m" + limits: + memory: "50Mi" + cpu: "100m" volumeMounts: - name: cnibin mountPath: {{ .Values.config.scMountPaths.cnibin }} @@ -55,4 +66,3 @@ spec: hostPath: path: /opt/cni/bin {{- end }} -{{- end }} diff --git a/sriov/templates/daemonSet_sriov_device_plugin.yaml b/sriov/templates/daemonSet_sriov_device_plugin.yaml index 9f24eb8..10fc28b 100644 --- a/sriov/templates/daemonSet_sriov_device_plugin.yaml +++ b/sriov/templates/daemonSet_sriov_device_plugin.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "16" ) }} -{{- if .Values.manifests.daemonSet_sriov_device_plugin }} apiVersion: apps/v1 kind: DaemonSet metadata: @@ -23,8 +22,8 @@ metadata: spec: selector: matchLabels: -{{- include "sriov-dp.serviceAccount.Name" . | indent 6 }} -{{- include "sriov-dp.labels" . | indent 6 }} + {{- include "sriov-dp.serviceAccount.Name" . | indent 6 }} + {{- include "sriov-dp.labels" . | indent 6 }} updateStrategy: type: RollingUpdate rollingUpdate: @@ -32,26 +31,34 @@ spec: template: metadata: labels: -{{- include "sriov-dp.serviceAccount.Name" . | indent 8 }} -{{- include "sriov-dp.labels" . | indent 8 }} + {{- include "sriov-dp.serviceAccount.Name" . | indent 8 }} + {{- include "sriov-dp.labels" . | indent 8 }} annotations: checksum/config: {{ include (print $.Template.BasePath "/configMap_sriov_device_plugin.yaml") . | sha256sum }} spec: priorityClassName: system-node-critical hostNetwork: true - nodeSelector: {{- toYaml .Values.labels.nodeSelector | nindent 8 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} tolerations: -{{- include "sriov.tolerations" . | indent 6 }} + {{- toYaml . | nindent 8 }} + {{- end }} serviceAccountName: {{ .Values.serviceAccount.name }} containers: - name: kube-{{ .Chart.Name }}-dp image: {{ .Values.images.sriovDevicePlugin.repository }}:{{ .Values.images.sriovDevicePlugin.tag }} imagePullPolicy: {{ .Values.images.pullPolicy }} + {{- with .Values.args}} args: - - --log-dir=sriovdp - - --log-level=10 + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.securityContext }} securityContext: - privileged: {{ .Values.securityContext.privileged }} + {{- toYaml . | nindent 10 }} + {{- end }} volumeMounts: - name: devicesock mountPath: {{ .Values.config.sdpMountPaths.deviceSock }} @@ -72,4 +79,3 @@ spec: configMap: name: {{ .Release.Name }}-{{ .Chart.Name }}-{{ .Chart.Version }}-config {{- end }} -{{- end }} diff --git a/sriov/templates/net_attach_def_dpdk.yaml b/sriov/templates/net_attach_def_dpdk.yaml deleted file mode 100644 index 3a74586..0000000 --- a/sriov/templates/net_attach_def_dpdk.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2020 K8s Network Plumbing Group -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -{{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "16" ) }} -{{- if .Values.manifests.net_attach_def_dpdk }} -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: {{ .Values.config.netAttachDef.dpdkName }} - annotations: - k8s.v1.cni.cncf.io/resourceName: {{ .Values.config.netAttachDef.dpdkResourceName }} -spec: - config: |- -{{ toJson .Values.config.netAttachDef.dpdk | indent 4 }} -{{- end }} -{{- end }} diff --git a/sriov/templates/net_attach_def_netdev.yaml b/sriov/templates/net_attach_def_netdev.yaml deleted file mode 100644 index b01c7be..0000000 --- a/sriov/templates/net_attach_def_netdev.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2020 K8s Network Plumbing Group -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -{{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "16" ) }} -{{- if .Values.manifests.net_attach_def_netdev }} -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: {{ .Values.config.netAttachDef.netdeviceName }} - annotations: - k8s.v1.cni.cncf.io/resourceName: {{ .Values.config.netAttachDef.netdeviceResourceName }} -spec: - config: |- -{{ toJson .Values.config.netAttachDef.netdevice | indent 4 }} -{{- end }} -{{- end }} diff --git a/sriov/values.yaml b/sriov/values.yaml index 685ef2a..afd4a08 100644 --- a/sriov/values.yaml +++ b/sriov/values.yaml @@ -12,78 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Default values for sriov. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -#replicaCount: 1 - images: registry: docker.io sriovCni: - repository: nfvpe/sriov-cni - tag: v2.2 + repository: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin + tag: latest sriovDevicePlugin: repository: nfvpe/sriov-device-plugin tag: v3.1 pullPolicy: IfNotPresent -#imagePullSecrets: [] -#nameOverride: "" -#fullnameOverride: "" - serviceAccount: - # Specifies whether a service account should be created - #create: true - # Annotations to add to the service account - #annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template name: sriov-device-plugin-test -pod: - resources: - enabled: true - sriov_cni: - requests: - memory: "50Mi" - cpu: "100m" - limits: - memory: "50Mi" - cpu: "100m" -#podSecurityContext: {} - # fsGroup: 2000 +resources: + requests: + memory: "50Mi" + cpu: "100m" + limits: + memory: "50Mi" + cpu: "100m" securityContext: privileged: true - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 -#service: - #type: ClusterIP - #port: 80 +nodeSelector: + kubernetes.io/arch: amd64 -#ingress: - #enabled: false - #annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - #hosts: - #- host: chart-example.local - # paths: [] - #tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -labels: - nodeSelector: - kubernetes.io/arch: amd64 +# -- Arguments for the sriov-device-plugin +args: + - --log-dir=sriovdp + - --log-level=10 config: scMountPaths: @@ -92,74 +52,69 @@ config: deviceSock: "/var/lib/kubelet/device-plugins/" log: "/var/log" configVolume: "/etc/pcidp/config.json" - sriov_device_plugin: - resourceList: - - resourceName: intel_sriov_netdevice - selectors: - vendors: - - '8086' - devices: - - 154c - - 10ed - drivers: - - i40evf - - ixgbevf - - resourceName: intel_sriov_dpdk - selectors: - vendors: - - '8086' - devices: - - 154c - - 10ed - drivers: - - vfio-pci - pfNames: - - enp67s0f1#8-31 - - enp68s0f0#8-31 - - resourceName: mlnx_sriov_rdma - isRdma: true - selectors: - vendors: - - 15b3 - devices: - - '1018' - drivers: - - mlx5_ib - netAttachDef: - dpdkName: 30-intel-sriov-dpdk - dpdkResourceName: intel.com/intel_sriov_dpdk - dpdk: - type: sriov - cniVersion: 0.3.1 - name: sriov-dpdk-network - vlan: 1000 - spoofchk: "off" - trust: "on" - netdeviceName: 20-intel-sriov-netdevice - netdeviceResourceName: intel.com/intel_sriov_netdevice - netdevice: - type: sriov - cniVersion: 0.3.1 - name: sriov-network - ipam: - type: host-local - subnet: 10.10.10.0/24 - rangeStart: 10.10.10.51 - rangeEnd: 10.10.10.100 - routes: - - dst: 0.0.0.0/0 - gateway: 10.10.10.1 + sriov_device_plugin: | + { + "resourceList": [ + { + "resourceName": "intel_sriov_netdevice", + "selectors": { + "vendors": [ + "8086" + ], + "devices": [ + 154c, + 10ed + ], + "drivers": [ + "i40evf", + "ixgbevf" + ] + } + }, + { + "resourceName": "intel_sriov_dpdk", + "selectors": { + "vendors": [ + "8086" + ], + "devices": [ + 154c, + 10ed + ], + "drivers": [ + "vfio-pci" + ] + }, + "pfNames": [ + "enp67s0f1#8-31", + "enp68s0f0#8-31" + ] + }, + { + "resourceName": "mlnx_sriov_rdma", + "isRdma": true, + "selectors": { + "vendors": [ + 15b3 + ], + "devices": [ + "1018" + ], + "drivers": [ + "mlx5_ib" + ] + } + } + ] + } + -manifests: - serviceAccount: true - configMap_sriov_device_plugin: true - daemonSet_sriov_device_plugin: true - daemonSet_sriov_cni: true - net_attach_def_netdev: true - net_attach_def_dpdk: true - test_netdevice: true - test_dpdk: true +# -- Tolerations for pod assignment. +tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute -#tolerations: [] -#affinity: {} +affinity: {}