Make OpenEBS as a helm extension

OpenEBS is migrated from `spec.extensions.storage` to helm. This is done for consistency with the rest of the documentation and allows to untie the OpenEBS version from the k0s version. This commit makes three changes: 1- Add the corresponding docs changes 2- Deprecate the storage API in kubebuilder 3- Force a panic if OpenEBS is defined as both a helm extension and a storage extension Signed-off-by: Juan-Luis de Sousa-Valadas Castaño <[email protected]>
k0sproject · Dec 12, 2023 · 07bb4da · 07bb4da
1 parent 697de73
commit 07bb4da
Show file tree

Hide file tree

Showing 6 changed files with 289 additions and 41 deletions.
diff --git a/docs/examples/openebs.md b/docs/examples/openebs.md
@@ -0,0 +1,161 @@
+# OpenEBS
+
+This tutorial covers the installation of OpenEBS as a helm chart, both from
+scratch and how to migrate it to a helm extension.
+
+## Installing OpenEBS from scratch
+
+**WARNING**: Do not configure OpenEBS both as a storage extension and helm
+extension. Because this is considered an invalid configuration k0s will ignore
+the configuration file entirely in order to prevent accidental upgrades or
+downgrades. The chart objects defined in the API will still have normal behavior.
+
+OpenEBS can be installed as a helm chart by adding it as an extension to your configuration:
+
+```yaml
+  extensions:
+    helm:
+      repositories:
+      - name: openebs-internal
+        url: https://openebs.github.io/charts
+      charts:
+      - name: openebs
+        chartname: openebs-internal/openebs
+        version: "3.9.0"
+        namespace: openebs
+        order: 1
+        values: |
+          localprovisioner:
+            hostpathClass:
+              enabled: true
+              isDefaultClass: false
+```
+
+If you want OpenEBS to be your default storage class set `isDefaultClass` to `true`.
+
+## Migrating bundled OpenEBS to helm extension
+
+The bundled OpenEBS extension is already a helm extension installed as a
+`chart.helm.k0sproject.io` object in the kubernetes apiserver.
+
+For this reason all we need to do is modifying the storage in the configuration
+storage type to `external_storage` instead of `openebs_local_storage`.
+
+If you are using [dynamic configuration](https://github.com/k0sproject/k0s/pull/3651) you can run this command:
+
+```shell
+kubectl patch clusterconfig -n kube-system  k0s --patch '{"spec":{"extensions":{"storage":{"type":"external_storage"}}}}' --type=merge
+```
+
+If you are using a configuration replace `spec.extensions.storage.type` from
+`openebs_local_storage` to `external_storage` in all control plane nodes.
+
+For clusters configured with a file ratehr than dynamic config it's
+advised to make sure that the configuration file is correct, although
+it's not required.
+
+Once the configuration is set to `external_storage` it the object can be
+managed as a chart:
+
+```shell
+kubectl get chart -n kube-system k0s-addon-chart-openebs -o yaml
+```
+
+Once migrated, it's recommended to perform an upgrade to 3.9.0:
+
+```shell
+kubectl patch chart -n kube-system  k0s-addon-chart-openebs --patch '{"spec":{"version":"3.9.0"}}' --type=merge
+```
+
+Optionally, if it's prefered to have the OpenEBS configuration as part of the
+cluster configuration, it may be added it to the configuration file or cluster
+configuration object following the steps of installing it from scratch. As long
+as the values in the file match with the ones in the chart object this will not
+trigger any changes.
+
+## Usage
+
+Once installed The cluster will have two storage classes available for you to use:
+
+```shell
+k0s kubectl get storageclass
+```
+
+```shell
+NAME               PROVISIONER        RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
+openebs-device     openebs.io/local   Delete          WaitForFirstConsumer   false                  24s
+openebs-hostpath   openebs.io/local   Delete          WaitForFirstConsumer   false                  24s
+```
+
+The `openebs-hostpath` is the storage class that maps to the `/var/openebs/local`
+
+The `openebs-device` is not configured and could be configured by [manifest deployer](../manifests.md) accordingly to the [OpenEBS documentation](https://docs.openebs.io/)
+
+### Example
+
+Use following manifests as an example of pod with mounted volume:
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nginx-pvc
+  namespace: default
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: openebs-hostpath
+  resources:
+    requests:
+      storage: 5Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+  namespace: default
+  labels:
+    app: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - image: nginx
+        name: nginx
+        volumeMounts:
+        - name: persistent-storage
+          mountPath: /var/lib/nginx
+      volumes:
+      - name: persistent-storage
+        persistentVolumeClaim:
+          claimName: nginx-pvc
+```
+
+```shell
+k0s kubectl apply -f nginx.yaml
+```
+
+```shell
+persistentvolumeclaim/nginx-pvc created
+deployment.apps/nginx created
+bash-5.1# k0s kc get pods
+NAME                    READY   STATUS    RESTARTS   AGE
+nginx-d95bcb7db-gzsdt   1/1     Running   0          30s
+```
+
+```shell
+k0s kubectl get pv
+```
+
+```shell
+NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM               STORAGECLASS       REASON   AGE
+pvc-9a7fae2d-eb03-42c3-aaa9-1a807d5df12f   5Gi        RWO            Delete           Bound    default/nginx-pvc   openebs-hostpath            30s
+```
diff --git a/docs/storage.md b/docs/storage.md
@@ -2,13 +2,49 @@
 
 k0s supports any volume provider that implements the [CSI specification](https://github.com/container-storage-interface/spec). For convenience, k0s comes bundled in with support for [OpenEBS local path provisioner](https://openebs.io/docs/concepts/localpv).
 
-The choise of which CSI provider to use depends heavily on the use case and infrastructure you're running on and the use case you have.
+The choice of which CSI provider to use depends heavily on the use case and infrastructure you're running on and the use case you have.
 
-## Bundled OpenEBS storage
+## CSI
+
+k0s supports a wide range of different storage options by utilizing Container Storage Interface (CSI). All Kubernetes storage solutions are supported and users can easily select the storage that fits best for their needs.
+
+When the storage solution implements Container Storage Interface (CSI), containers can communicate with the storage for creation and configuration of persistent volumes. This makes it easy to dynamically provision the requested volumes. It also expands the supported storage solutions from the previous generation, in-tree volume plugins. More information about the CSI concept is described on the [Kubernetes Blog](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/).
+
+![k0s storage](img/k0s_storage.png)
+
+### Installing 3rd party storage solutions
+
+Follow your storage driver's installation instructions. Note that the Kubelet installed by k0s uses a slightly different path for its working directory (`/varlib/k0s/kubelet` instead of `/var/lib/kubelet`). Consult the CSI driver's configuration documentation on how to customize this path.
+
+## Example storage solutions
+
+Different Kubernetes storage solutions are explained in the [official Kubernetes storage documentation](https://kubernetes.io/docs/concepts/storage/volumes/). All of them can be used with k0s. Here are some popular ones:
+
+- Rook-Ceph (Open Source)
+- MinIO (Open Source)
+- Gluster (Open Source)
+- Longhorn (Open Source)
+- Amazon EBS
+- Google Persistent Disk
+- Azure Disk
+- Portworx
+
+If you are looking for a fault-tolerant storage with data replication, you can find a k0s tutorial for configuring Ceph storage with Rook [in here](examples/rook-ceph.md).
+
+## Bundled OpenEBS storage (deprecated)
+
+Bundled OpenEBS was deprecated in favor of running it [as a helm extension](./examples/openebs.md),
+this documentation is maintained as a reference for existing installations.
+
+This was done for two reasons:
 
-K0s comes out with bundled OpenEBS installation which can be enabled by using [configuration file](./configuration.md)
+1. By installing it as a helm extenssion the users have more control and flexibility without adding complexity.
+2. Allows users to choose the OpenEBS version independently of their k0s version.
+3. Makes k0s configuration more consistent.
 
-Use following configuration as an example:
+For new installations or to migrate existing installations refer to the [OpenEBS extension page](./examples/openebs.md).
+
+OpenEBS extension is enabled by setting spec.extensions.storage.type to `openebs_local_storage`:
 
 ```yaml
 spec:
@@ -101,30 +137,3 @@ k0s kubectl get pv
 NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM               STORAGECLASS       REASON   AGE
 pvc-9a7fae2d-eb03-42c3-aaa9-1a807d5df12f   5Gi        RWO            Delete           Bound    default/nginx-pvc   openebs-hostpath            30s
 ```
-
-## CSI
-
-k0s supports a wide range of different storage options by utilizing Container Storage Interface (CSI). All Kubernetes storage solutions are supported and users can easily select the storage that fits best for their needs.
-
-When the storage solution implements Container Storage Interface (CSI), containers can communicate with the storage for creation and configuration of persistent volumes. This makes it easy to dynamically provision the requested volumes. It also expands the supported storage solutions from the previous generation, in-tree volume plugins. More information about the CSI concept is described on the [Kubernetes Blog](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/).
-
-![k0s storage](img/k0s_storage.png)
-
-### Installing 3rd party storage solutions
-
-Follow your storage driver's installation instructions. Note that the Kubelet installed by k0s uses a slightly different path for its working directory (`/varlib/k0s/kubelet` instead of `/var/lib/kubelet`). Consult the CSI driver's configuration documentation on how to customize this path.
-
-## Example storage solutions
-
-Different Kubernetes storage solutions are explained in the [official Kubernetes storage documentation](https://kubernetes.io/docs/concepts/storage/volumes/). All of them can be used with k0s. Here are some popular ones:
-
-- Rook-Ceph (Open Source)
-- MinIO (Open Source)
-- Gluster (Open Source)
-- Longhorn (Open Source)
-- Amazon EBS
-- Google Persistent Disk
-- Azure Disk
-- Portworx
-
-If you are looking for a fault-tolerant storage with data replication, you can find a k0s tutorial for configuring Ceph storage with Rook [in here](examples/rook-ceph.md).
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -62,6 +62,7 @@ nav:
       - Ambassador API Gateway: examples/ambassador-ingress.md
       - Ceph Storage with Rook: examples/rook-ceph.md
       - GitOps with Flux: examples/gitops-flux.md
+      - OpenEBS storage: examples/openebs.md
   - Troubleshooting:
       - FAQ: FAQ.md
       - Dockershim deprecation: dockershim.md

diff --git a/pkg/apis/k0s/v1beta1/extensions.go b/pkg/apis/k0s/v1beta1/extensions.go
@@ -28,6 +28,8 @@ var _ Validateable = (*ClusterExtensions)(nil)
 
 // ClusterExtensions specifies cluster extensions
 type ClusterExtensions struct {
+	//+kubebuilder:deprecatedversion:warning="storage is deprecated and will be removed in 1.30. https://docs.k0sproject.io/stable/examples/openebs".
+	//
 	Storage *StorageExtension `json:"storage"`
 	Helm    *HelmExtensions   `json:"helm"`
 }

diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go
@@ -82,15 +82,9 @@ func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0
 	ec.L.Info("Extensions reconciliation started")
 	defer ec.L.Info("Extensions reconciliation finished")
 
-	helmSettings := clusterConfig.Spec.Extensions.Helm
-	var err error
-	switch clusterConfig.Spec.Extensions.Storage.Type {
-	case k0sAPI.OpenEBSLocal:
-		helmSettings, err = addOpenEBSHelmExtension(helmSettings, clusterConfig.Spec.Extensions.Storage)
-		if err != nil {
-			ec.L.WithError(err).Error("Can't add openebs helm extension")
-		}
-	default:
+	helmSettings, err := ec.configureStorage(clusterConfig)
+	if err != nil {
+		return fmt.Errorf("Cannot configure storage: %w", err)
 	}
 
 	if err := ec.reconcileHelmExtensions(helmSettings); err != nil {
@@ -100,6 +94,24 @@ func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0
 	return nil
 }
 
+func (ec *ExtensionsController) configureStorage(clusterConfig *k0sAPI.ClusterConfig) (*k0sAPI.HelmExtensions, error) {
+	helmSettings := clusterConfig.Spec.Extensions.Helm
+	if clusterConfig.Spec.Extensions.Storage.Type != k0sAPI.OpenEBSLocal {
+		return helmSettings, nil
+	}
+
+	for _, chart := range helmSettings.Charts {
+		if chart.ChartName == "openebs-internal/openebs" {
+			return nil, fmt.Errorf("openebs-internal/openebs is defined in spec.extensions.helm.charts and spec.extensions.storage.type is set to openebs_local_storage. https://docs.k0sproject.io/stable/examples/openebs")
+		}
+	}
+	helmSettings, err := addOpenEBSHelmExtension(helmSettings, clusterConfig.Spec.Extensions.Storage)
+	if err != nil {
+		return nil, fmt.Errorf("Can't add openebs helm extension")
+	}
+	return helmSettings, nil
+}
+
 func addOpenEBSHelmExtension(helmSpec *k0sAPI.HelmExtensions, storageExtension *k0sAPI.StorageExtension) (*k0sAPI.HelmExtensions, error) {
 	openEBSValues := map[string]interface{}{
 		"localprovisioner": map[string]interface{}{

diff --git a/pkg/component/controller/extensions_controller_test.go b/pkg/component/controller/extensions_controller_test.go
@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/k0sproject/k0s/pkg/apis/helm/v1beta1"
+	k0sAPI "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -128,9 +129,71 @@ func TestChartNeedsUpgrade(t *testing.T) {
 
 	cr := new(ChartReconciler)
 	for _, tc := range testCases {
-		t.Run("", func(t *testing.T) {
+		t.Run(tc.description, func(t *testing.T) {
 			actual := cr.chartNeedsUpgrade(tc.chart)
 			assert.Equal(t, tc.expected, actual)
 		})
 	}
 }
+
+func addHelmExtension(config *k0sAPI.ClusterConfig) *k0sAPI.ClusterConfig {
+	config.Spec.Extensions.Storage.Type = k0sAPI.OpenEBSLocal
+	return config
+}
+func addStorageExtension(config *k0sAPI.ClusterConfig) *k0sAPI.ClusterConfig {
+	config.Spec.Extensions.Helm, _ = addOpenEBSHelmExtension(config.Spec.Extensions.Helm, config.Spec.Extensions.Storage)
+	return config
+}
+
+func TestConfigureStorage(t *testing.T) {
+	var testCases = []struct {
+		description     string
+		clusterConfig   *k0sAPI.ClusterConfig
+		expectedErr     bool
+		expectedOpenEBS bool
+	}{
+		{
+			"no_openebs",
+			k0sAPI.DefaultClusterConfig(),
+			false,
+			false,
+		},
+		{
+			"openebs_helm_extension",
+			addHelmExtension(k0sAPI.DefaultClusterConfig()),
+			false,
+			true,
+		},
+		{
+			"openebs_storage_extension",
+			addStorageExtension(k0sAPI.DefaultClusterConfig()),
+			false,
+			true,
+		},
+		{
+			"openebs_both",
+			addStorageExtension(addHelmExtension(k0sAPI.DefaultClusterConfig())),
+			true,
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			ec := ExtensionsController{}
+			helmSettings, err := ec.configureStorage(tc.clusterConfig)
+
+			if tc.expectedErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+
+			if tc.expectedOpenEBS {
+				assert.Equal(t, "openebs", helmSettings.Charts[0].Name)
+				assert.Equal(t, 1, len(helmSettings.Charts))
+			}
+		})
+	}
+
+}