Gracefull shutdown

Cleanup stage volumes on shutdown Allow to disable useless csi-attacher Code refactoring Helm chart refactoring
seaweedfs · Aug 5, 2023 · b07d4c6 · b07d4c6
1 parent d777b87
commit b07d4c6
Show file tree

Hide file tree

Showing 16 changed files with 768 additions and 3,045 deletions.
diff --git a/cmd/seaweedfs-csi-driver/main.go b/cmd/seaweedfs-csi-driver/main.go
@@ -12,6 +12,10 @@ import (
 )
 
 var (
+	runNode        = flag.Bool("node", false, "run node server")
+	runController  = flag.Bool("controller", false, "run controller server")
+	enableAttacher = flag.Bool("attacher", false, "enable attacher")
+
 	filer             = flag.String("filer", "localhost:8888", "filer server")
 	endpoint          = flag.String("endpoint", "unix://tmp/seaweedfs-csi.sock", "CSI endpoint to accept gRPC calls")
 	nodeID            = flag.String("nodeid", "", "node id")
@@ -53,7 +57,11 @@ func main() {
 
 	glog.Infof("connect to filer %s", *filer)
 
-	drv := driver.NewSeaweedFsDriver(*filer, *nodeID, *endpoint)
+	drv := driver.NewSeaweedFsDriver(*filer, *nodeID, *endpoint, *enableAttacher)
+
+	drv.RunNode = *runNode
+	drv.RunController = *runController
+
 	drv.ConcurrentWriters = *concurrentWriters
 	drv.CacheCapacityMB = *cacheCapacityMB
 	drv.CacheDir = *cacheDir

diff --git a/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yml b/deploy/helm/seaweedfs-csi-driver/templates/csidriver.yml
@@ -3,5 +3,5 @@ kind: CSIDriver
 metadata:
   name: {{ .Values.driverName }}
 spec:
-  attachRequired: true
+  attachRequired: {{ .Values.csiAttacher.enabled }}
   podInfoOnMount: true
diff --git a/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml b/deploy/helm/seaweedfs-csi-driver/templates/daemonset.yml
@@ -31,43 +31,20 @@ spec:
       tolerations: {{ toYaml . | nindent 8 }}
       {{- end }}
       containers:
-        - name: driver-registrar
-          image: {{ .Values.csiNodeDriverRegistrar.image }}
-          imagePullPolicy: {{ .Values.imagePullPolicy }}
-          args:
-            - "--v=5"
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
-          env:
-            - name: ADDRESS
-              value: /csi/csi.sock
-            - name: DRIVER_REG_SOCK_PATH
-              value: {{ .Values.node.volumes.plugins_dir }}/{{ .Values.driverName }}/csi.sock
-            - name: KUBE_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          resources:
-            {{ toYaml .Values.csiNodeDriverRegistrar.resources | nindent 12 }}
-          volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi/
-            - name: registration-dir
-              mountPath: /registration/
         - name: csi-seaweedfs-plugin
-          securityContext:
-            {{ toYaml .Values.seaweedfsCsiPlugin.securityContext | nindent 12 }}
+          securityContext: {{ toYaml .Values.seaweedfsCsiPlugin.securityContext | nindent 12 }}
           image: {{.Values.seaweedfsCsiPlugin.image }}
           imagePullPolicy: {{ .Values.imagePullPolicy }}
           args:
-            - "--endpoint=$(CSI_ENDPOINT)"
-            - "--filer=$(SEAWEEDFS_FILER)"
-            - "--nodeid=$(NODE_ID)"
-            - "--cacheDir=/var/cache/seaweedfs"
-            - "--dataLocality={{ .Values.dataLocality }}"
+            - --endpoint=$(CSI_ENDPOINT)
+            - --filer=$(SEAWEEDFS_FILER)
+            - --nodeid=$(NODE_ID)
+            - --cacheDir=/var/cache/seaweedfs
+            - --dataLocality={{ .Values.dataLocality }}
             {{- if .Values.node.injectTopologyInfoFromNodeLabel.enabled }}
-            - "--dataCenter=$(DATACENTER)"
+            - --dataCenter=$(DATACENTER)
             {{- end }}
+            - --node
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
@@ -92,12 +69,32 @@ spec:
             - name: WEED_GRPC_CA
               value: /var/run/secrets/app/tls/ca.crt
             {{- end }}
-            {{- if .Values.logVerbosity }}
+            {{- with .Values.logVerbosity }}
             - name: WEED_V
-              value: {{ .Values.logVerbosity | quote }}
+              value: {{ . | quote }}
+            {{- end }}
+          ports:
+            - containerPort: 9808
+              name: healthz
+              protocol: TCP
+          {{- with .Values.node.livenessProbe }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            {{- with .failureThreshold }}
+            failureThreshold: {{ . }}
+            {{- end }}
+            {{- with .initialDelaySeconds }}
+            initialDelaySeconds: {{ . }}
+            {{- end }}
+            {{- with .timeoutSeconds }}
+            timeoutSeconds: {{ . }}
             {{- end }}
-          resources:
-            {{ toYaml .Values.seaweedfsCsiPlugin.resources | nindent 12 }}
+            {{- with .periodSeconds }}
+            periodSeconds: {{ . }}
+            {{- end }}
+          {{- end }}
           volumeMounts:
             - name: plugin-dir
               mountPath: /csi
@@ -115,6 +112,74 @@ spec:
             {{- end }}
             - name: cache
               mountPath: /var/cache/seaweedfs
+          resources: {{ toYaml .Values.node.resources | nindent 12 }}
+
+        # driver registrar
+        - name: driver-registrar
+          image: {{ .Values.csiNodeDriverRegistrar.image }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --http-endpoint=:9809
+            #- --v=5
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: {{ .Values.node.volumes.plugins_dir }}/{{ .Values.driverName }}/csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          {{- with .Values.csiNodeDriverRegistrar.livenessProbe }}
+          ports:
+            - containerPort: 9809
+              name: healthz
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: healthz
+            {{- with .failureThreshold }}
+            failureThreshold: {{ . }}
+            {{- end }}
+            {{- with .initialDelaySeconds }}
+            initialDelaySeconds: {{ . }}
+            {{- end }}
+            {{- with .timeoutSeconds }}
+            timeoutSeconds: {{ . }}
+            {{- end }}
+            {{- with .periodSeconds }}
+            periodSeconds: {{ . }}
+            {{- end }}
+          {{- end }}
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi/
+            - name: registration-dir
+              mountPath: /registration/
+          resources: {{ toYaml .Values.csiNodeDriverRegistrar.resources | nindent 12 }}
+
+        # liveness probe
+        {{- if .Values.node.livenessProbe }}
+        - name: csi-liveness-probe
+          image: {{ .Values.csiLivenessProbe.image }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --http-endpoint=:9808
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+          ports:
+            - containerPort: 9808
+              name: livenessprobe
+          volumeMounts:
+            - name: plugin-dir
+              mountPath: /csi
+          resources: {{ toYaml .Values.csiLivenessProbe.resources | nindent 12 }}
+        {{- end }}
+
       volumes:
         - name: registration-dir
           hostPath:

diff --git a/deploy/helm/seaweedfs-csi-driver/templates/rbac.yml b/deploy/helm/seaweedfs-csi-driver/templates/rbac.yml
@@ -31,7 +31,6 @@ rules:
   - apiGroups: [ "" ]
     resources: [ "pods" ]
     verbs: [ "get", "list", "watch" ]
-
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -45,6 +44,8 @@ roleRef:
   kind: ClusterRole
   name: {{ template "seaweedfs-csi-driver.name" . }}-provisioner-role
   apiGroup: rbac.authorization.k8s.io
+
+{{- if .Values.csiAttacher.enabled }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -63,7 +64,6 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["volumeattachments", "volumeattachments/status"]
     verbs: ["get", "list", "watch", "update", "patch"]
-
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -77,6 +77,9 @@ roleRef:
   kind: ClusterRole
   name: {{ template "seaweedfs-csi-driver.name" . }}-attacher-role
   apiGroup: rbac.authorization.k8s.io
+{{- end }}
+
+{{- if .Values.csiSnapshotter.enabled }}
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -123,6 +126,8 @@ roleRef:
   kind: ClusterRole
   name: {{ template "seaweedfs-csi-driver.name" . }}-snapshotter-role
   apiGroup: rbac.authorization.k8s.io
+{{- end }}
+
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -148,6 +153,7 @@ roleRef:
   kind: ClusterRole
   name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-controller-role
   apiGroup: rbac.authorization.k8s.io
+
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -176,3 +182,28 @@ roleRef:
   kind: ClusterRole
   name: {{ template "seaweedfs-csi-driver.name" . }}-driver-registrar-node-role
   apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "seaweedfs-csi-driver.name" . }}-leader-election-controller-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "seaweedfs-csi-driver.name" . }}-leader-election-controller-binding
+  namespace: {{ .Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    namespace: {{ .Release.Namespace }}
+    name: {{ template "seaweedfs-csi-driver.name" . }}-controller-sa
+roleRef:
+  kind: Role
+  name: {{ template "seaweedfs-csi-driver.name" . }}-leader-election-controller-role
+  apiGroup: rbac.authorization.k8s.io