Skip to content

Commit

Permalink
permit specifying daemonset and deployment resources
Browse files Browse the repository at this point in the history 
closes NetApp#853

Signed-off-by: Clément Nussbaumer <[email protected]>
  • Loading branch information
@clementnuss
clementnuss committed Oct 17, 2024
1 parent 77449b3 commit 0169eb0
Show file tree
Hide file tree
Showing 7 changed files with 230 additions and 126 deletions.
114 changes: 58 additions & 56 deletions cli/k8s_client/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,65 +143,67 @@ type KubernetesClient interface {
}

type DeploymentYAMLArguments struct {
DeploymentName string `json:"deploymentName"`
TridentImage string `json:"tridentImage"`
AutosupportImage string `json:"autosupportImage"`
AutosupportProxy string `json:"autosupportProxy"`
AutosupportInsecure bool `json:"autosupportInsecure"`
AutosupportCustomURL string `json:"autosupportCustomURL"`
AutosupportSerialNumber string `json:"autosupportSerialNumber"`
AutosupportHostname string `json:"autosupportHostname"`
ImageRegistry string `json:"imageRegistry"`
LogFormat string `json:"logFormat"`
LogLevel string `json:"logLevel"`
LogWorkflows string `json:"logWorkflows"`
LogLayers string `json:"logLayers"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
Debug bool `json:"debug"`
UseIPv6 bool `json:"useIPv6"`
SilenceAutosupport bool `json:"silenceAutosupport"`
Version *versionutils.Version `json:"version"`
DisableAuditLog bool `json:"disableAuditLog"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
ServiceAccountName string `json:"serviceAccountName"`
ImagePullPolicy string `json:"imagePullPolicy"`
EnableForceDetach bool `json:"enableForceDetach"`
ACPImage string `json:"acpImage"`
EnableACP bool `json:"enableACP"`
CloudProvider string `json:"cloudProvider"`
IdentityLabel bool `json:"identityLabel"`
K8sAPIQPS int `json:"k8sAPIQPS"`
DeploymentName string `json:"deploymentName"`
TridentImage string `json:"tridentImage"`
AutosupportImage string `json:"autosupportImage"`
AutosupportProxy string `json:"autosupportProxy"`
AutosupportInsecure bool `json:"autosupportInsecure"`
AutosupportCustomURL string `json:"autosupportCustomURL"`
AutosupportSerialNumber string `json:"autosupportSerialNumber"`
AutosupportHostname string `json:"autosupportHostname"`
ImageRegistry string `json:"imageRegistry"`
LogFormat string `json:"logFormat"`
LogLevel string `json:"logLevel"`
LogWorkflows string `json:"logWorkflows"`
LogLayers string `json:"logLayers"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
Debug bool `json:"debug"`
UseIPv6 bool `json:"useIPv6"`
SilenceAutosupport bool `json:"silenceAutosupport"`
Version *versionutils.Version `json:"version"`
DisableAuditLog bool `json:"disableAuditLog"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
ServiceAccountName string `json:"serviceAccountName"`
ImagePullPolicy string `json:"imagePullPolicy"`
EnableForceDetach bool `json:"enableForceDetach"`
ACPImage string `json:"acpImage"`
EnableACP bool `json:"enableACP"`
CloudProvider string `json:"cloudProvider"`
IdentityLabel bool `json:"identityLabel"`
K8sAPIQPS int `json:"k8sAPIQPS"`
Resources *v1.ResourceRequirements `json:"resources"`
}

type DaemonsetYAMLArguments struct {
DaemonsetName string `json:"daemonsetName"`
TridentImage string `json:"tridentImage"`
ImageRegistry string `json:"imageRegistry"`
KubeletDir string `json:"kubeletDir"`
LogFormat string `json:"logFormat"`
LogLevel string `json:"logLevel"`
LogWorkflows string `json:"logWorkflows"`
LogLayers string `json:"logLayers"`
ProbePort string `json:"probePort"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
EnableForceDetach bool `json:"enableForceDetach"`
DisableAuditLog bool `json:"disableAuditLog"`
Debug bool `json:"debug"`
Version *versionutils.Version `json:"version"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
ServiceAccountName string `json:"serviceAccountName"`
ImagePullPolicy string `json:"imagePullPolicy"`
ISCSISelfHealingInterval string `json:"iscsiSelfHealingInterval"`
ISCSISelfHealingWaitTime string `json:"iscsiSelfHealingWaitTime"`
NodePrep []string `json:"nodePrep"`
DaemonsetName string `json:"daemonsetName"`
TridentImage string `json:"tridentImage"`
ImageRegistry string `json:"imageRegistry"`
KubeletDir string `json:"kubeletDir"`
LogFormat string `json:"logFormat"`
LogLevel string `json:"logLevel"`
LogWorkflows string `json:"logWorkflows"`
LogLayers string `json:"logLayers"`
ProbePort string `json:"probePort"`
ImagePullSecrets []string `json:"imagePullSecrets"`
Labels map[string]string `json:"labels"`
ControllingCRDetails map[string]string `json:"controllingCRDetails"`
EnableForceDetach bool `json:"enableForceDetach"`
DisableAuditLog bool `json:"disableAuditLog"`
Debug bool `json:"debug"`
Version *versionutils.Version `json:"version"`
HTTPRequestTimeout string `json:"httpRequestTimeout"`
NodeSelector map[string]string `json:"nodeSelector"`
Tolerations []map[string]string `json:"tolerations"`
ServiceAccountName string `json:"serviceAccountName"`
ImagePullPolicy string `json:"imagePullPolicy"`
ISCSISelfHealingInterval string `json:"iscsiSelfHealingInterval"`
ISCSISelfHealingWaitTime string `json:"iscsiSelfHealingWaitTime"`
NodePrep []string `json:"nodePrep"`
Resources *v1.ResourceRequirements `json:"resources"`
}

type TridentVersionPodYAMLArguments struct {
Expand Down
25 changes: 22 additions & 3 deletions cli/k8s_client/yaml_factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
commonconfig "github.com/netapp/trident/config"
. "github.com/netapp/trident/logging"
"github.com/netapp/trident/utils"
v1 "k8s.io/api/core/v1"
"sigs.k8s.io/yaml"
)

const (
Expand Down Expand Up @@ -524,6 +526,7 @@ func GetCSIDeploymentYAML(args *DeploymentYAMLArguments) string {
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "NODE_TOLERATIONS", constructTolerations(args.Tolerations))
deploymentYAML = utils.ReplaceMultilineYAMLTag(deploymentYAML, "RESOURCES", constructResources(args.Resources))
deploymentYAML = strings.ReplaceAll(deploymentYAML, "{ENABLE_FORCE_DETACH}", strconv.FormatBool(args.EnableForceDetach))
deploymentYAML = strings.ReplaceAll(deploymentYAML, "{ENABLE_ACP}", enableACP)
deploymentYAML = strings.ReplaceAll(deploymentYAML, "{K8S_API_CLIENT_TRIDENT_THROTTLE}", K8sAPITridentThrottle)
Expand Down Expand Up @@ -594,6 +597,7 @@ spec:
{ENABLE_ACP}
{DEBUG}
{K8S_API_CLIENT_TRIDENT_THROTTLE}
{RESOURCES}
livenessProbe:
exec:
command:
Expand Down Expand Up @@ -647,9 +651,7 @@ spec:
{AUTOSUPPORT_HOSTNAME}
{AUTOSUPPORT_DEBUG}
{AUTOSUPPORT_INSECURE}
resources:
limits:
memory: 1Gi
{RESOURCES}
volumeMounts:
- name: asup-dir
mountPath: /asup
Expand All @@ -667,6 +669,7 @@ spec:
- "--retry-interval-start=8s"
- "--retry-interval-max=30s"
{K8S_API_CLIENT_SIDECAR_THROTTLE}
{RESOURCES}
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
Expand All @@ -686,6 +689,7 @@ spec:
- "--retry-interval-start=10s"
- "--csi-address=$(ADDRESS)"
{K8S_API_CLIENT_SIDECAR_THROTTLE}
{RESOURCES}
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
Expand All @@ -700,6 +704,7 @@ spec:
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
{K8S_API_CLIENT_SIDECAR_THROTTLE}
{RESOURCES}
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
Expand All @@ -718,6 +723,7 @@ spec:
- "--timeout=300s"
- "--csi-address=$(ADDRESS)"
{K8S_API_CLIENT_SIDECAR_THROTTLE}
{RESOURCES}
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
Expand Down Expand Up @@ -909,6 +915,7 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "OWNER_REF", constructOwnerRef(args.ControllingCRDetails))
daemonSetYAML = utils.ReplaceMultilineYAMLTag(daemonSetYAML, "RESOURCES", constructResources(args.Resources))

// Log before secrets are inserted into YAML.
Log().WithField("yaml", daemonSetYAML).Trace("CSI Daemonset Linux YAML.")
Expand Down Expand Up @@ -1056,6 +1063,7 @@ spec:
timeoutSeconds: 5
periodSeconds: 10
initialDelaySeconds: 15
{RESOURCES}
env:
- name: KUBE_NODE_NAME
valueFrom:
Expand Down Expand Up @@ -1097,6 +1105,7 @@ spec:
- "--v={SIDECAR_LOG_LEVEL}"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(REGISTRATION_PATH)"
{RESOURCES}
env:
- name: ADDRESS
value: /plugin/csi.sock
Expand Down Expand Up @@ -2762,6 +2771,16 @@ func constructOwnerRef(ownerRef map[string]string) string {
return ownerRefData
}

func constructResources(res *v1.ResourceRequirements) string {
r := struct {
Resources *v1.ResourceRequirements `json:"resources"`
}{
Resources: res,
}
bytes, _ := yaml.Marshal(r)
return string(bytes)
}

func constructImagePullSecrets(imagePullSecrets []string) string {
var imagePullSecretsData string
if len(imagePullSecrets) > 0 {
Expand Down
49 changes: 49 additions & 0 deletions cli/k8s_client/yaml_factory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
pspv1beta1 "k8s.io/api/policy/v1beta1"
csiv1 "k8s.io/api/storage/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/netapp/trident/config"
Expand Down Expand Up @@ -514,6 +515,30 @@ func TestGetCSIDeploymentYAML_K8sAPIQPS(t *testing.T) {
}
}

func TestGetCSIDeploymentYAML_Resources(t *testing.T) {
args := &DeploymentYAMLArguments{Resources: &v1.ResourceRequirements{
Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("114m"), v1.ResourceMemory: resource.MustParse("256Mi")},
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("200m"), v1.ResourceMemory: resource.MustParse("512Mi")},
}}

expectedResourcesString := `
resources:
limits:
cpu: 114m
memory: 256Mi
requests:
cpu: 200m
memory: 512Mi
`
yamlData := GetCSIDeploymentYAML(args)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedResourcesString,
fmt.Sprintf("expected resources in final YAML: %s", yamlData))
}

func TestGetCSIDaemonSetYAMLLinux(t *testing.T) {
versions := []string{"1.21.0", "1.23.0", "1.25.0"}

Expand Down Expand Up @@ -859,6 +884,30 @@ func TestGetCSIDaemonSetYAMLLinux_Tolerations(t *testing.T) {
fmt.Sprintf("expected default tolerations to not appear in final YAML: %s", yamlData))
}

func TestGetCSIDaemonSetYAMLLinux_Resources(t *testing.T) {
args := &DaemonsetYAMLArguments{Resources: &v1.ResourceRequirements{
Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("114m"), v1.ResourceMemory: resource.MustParse("256Mi")},
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("200m"), v1.ResourceMemory: resource.MustParse("512Mi")},
}}

expectedResourcesString := `
resources:
limits:
cpu: 114m
memory: 256Mi
requests:
cpu: 200m
memory: 512Mi
`
yamlData := GetCSIDaemonSetYAMLLinux(args)
_, err := yaml.YAMLToJSON([]byte(yamlData))
if err != nil {
t.Fatalf("expected valid YAML, got %s", yamlData)
}
assert.Contains(t, yamlData, expectedResourcesString,
fmt.Sprintf("expected resources in final YAML: %s", yamlData))
}

func TestGetCSIDaemonSetYAMLWindows(t *testing.T) {
versions := []string{"1.21.0", "1.23.0", "1.25.0"}

Expand Down
4 changes: 4 additions & 0 deletions helm/trident-operator/templates/tridentorchestrator.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@ spec:
{{- if .Values.tridentDebug }}
debug: {{ .Values.tridentDebug }}
{{- end }}
daemonsetResources:
{{- toYaml .Values.tridentDaemonsetResources | nindent 4 }}
deploymentResources:
{{- toYaml .Values.tridentDeploymentResources | nindent 4 }}
{{- if .Values.tridentLogLevel }}
logLevel: {{ .Values.tridentLogLevel }}
{{- end }}
Expand Down
12 changes: 12 additions & 0 deletions helm/trident-operator/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,18 @@ tridentDisableAuditLog: true
# tridentDebug allows the log level of Trident to be set to debug
tridentDebug: false

# tridentDaemonsetResources permits specifying the resources for all containers of the trident-node daemonset. example:
# tridentDaemonsetResources:
# requests:
# cpu: 250m
# limits:
# cpu: 100m
# memory: 1Gi
tridentDaemonsetResources: {}

# tridentDeploymentResources permits specifying the resources for all containers of the trident-controller deployment.
tridentDeploymentResources:

# tridentLogWorkflows allows specific Trident workflows to be enabled for trace logging or log suppression.
tridentLogWorkflows: ""

Expand Down
13 changes: 13 additions & 0 deletions operator/controllers/orchestrator/installer/installer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ var (

nodePrep []string

daemonsetResources *v1.ResourceRequirements
deploymentResources *v1.ResourceRequirements

CRDnames = []string{
ActionMirrorUpdateCRDName,
ActionSnapshotRestoreCRDName,
Expand Down Expand Up @@ -391,6 +394,14 @@ func (i *Installer) setInstallationParams(

nodePrep = validation.FormatProtocols(cr.Spec.NodePrep)

if cr.Spec.DaemonsetResources != nil {
daemonsetResources = cr.Spec.DaemonsetResources
}

if cr.Spec.DeploymentResources != nil {
deploymentResources = cr.Spec.DeploymentResources
}

if cr.Spec.ProbePort != nil {
probePort = strconv.FormatInt(*cr.Spec.ProbePort, 10)
}
Expand Down Expand Up @@ -1526,6 +1537,7 @@ func (i *Installer) createOrPatchTridentDeployment(
EnableACP: enableACP,
IdentityLabel: identityLabel,
K8sAPIQPS: k8sAPIQPS,
Resources: deploymentResources,
}

newDeploymentYAML := k8sclient.GetCSIDeploymentYAML(deploymentArgs)
Expand Down Expand Up @@ -1606,6 +1618,7 @@ func (i *Installer) createOrPatchTridentDaemonSet(
ISCSISelfHealingInterval: iscsiSelfHealingInterval,
ISCSISelfHealingWaitTime: iscsiSelfHealingWaitTime,
NodePrep: nodePrep,
Resources: daemonsetResources,
}

var newDaemonSetYAML string
Expand Down
Loading

0 comments on commit 0169eb0

Please sign in to comment.