From 4af3f676ff5c188f584f1882dc178256dc31da61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan-Luis=20de=20Sousa-Valadas=20Casta=C3=B1o?= Date: Fri, 3 Nov 2023 13:51:06 +0100 Subject: [PATCH] Make OpenEBS as a helm extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenEBS is migrated from `spec.extensions.storage` to helm. This is done for consistency with the rest of the documentation and allows to untie the OpenEBS version from the k0s version. This commit makes three changes: 1- Add the corresponding docs changes 2- Deprecate the storage API in kubebuilder 3- Force a panic if OpenEBS is defined as both a helm extension and a storage extension Signed-off-by: Juan-Luis de Sousa-Valadas Castaño --- docs/examples/openebs.md | 161 ++++++++++++++++++ docs/storage.md | 71 ++++---- mkdocs.yml | 1 + pkg/apis/k0s/v1beta1/extensions.go | 2 + .../controller/extensions_controller.go | 30 +++- .../controller/extensions_controller_test.go | 65 ++++++- 6 files changed, 289 insertions(+), 41 deletions(-) create mode 100644 docs/examples/openebs.md diff --git a/docs/examples/openebs.md b/docs/examples/openebs.md new file mode 100644 index 000000000000..67dfb78a1d4b --- /dev/null +++ b/docs/examples/openebs.md @@ -0,0 +1,161 @@ +# OpenEBS + +This tutorial covers the installation of OpenEBS as a helm chart, both from +scratch and how to migrate it to a helm extension. + +## Installing OpenEBS from scratch + +**WARNING**: Do not configure OpenEBS both as a storage extension and helm +extension. Because this is considered an invalid configuration k0s will ignore +the configuration file entirely in order to prevent accidental upgrades or +downgrades. The chart objects defined in the API will still have normal behavior. + +OpenEBS can be installed as a helm chart by adding it as an extension to your configuration: + +```yaml + extensions: + helm: + repositories: + - name: openebs-internal + url: https://openebs.github.io/charts + charts: + - name: openebs + chartname: openebs-internal/openebs + version: "3.9.0" + namespace: openebs + order: 1 + values: | + localprovisioner: + hostpathClass: + enabled: true + isDefaultClass: false +``` + +If you want OpenEBS to be your default storage class set `isDefaultClass` to `true`. + +## Migrating bundled OpenEBS to helm extension + +The bundled OpenEBS extension is already a helm extension installed as a +`chart.helm.k0sproject.io` object in the kubernetes apiserver. + +For this reason all we need to do is modifying the storage in the configuration +storage type to `external_storage` instead of `openebs_local_storage`. + +If you are using [dynamic configuration](../dynamic-configuration.md) you can run this command: + +```shell +kubectl patch clusterconfig -n kube-system k0s --patch '{"spec":{"extensions":{"storage":{"type":"external_storage"}}}}' --type=merge +``` + +If you are using a configuration replace `spec.extensions.storage.type` from +`openebs_local_storage` to `external_storage` in all control plane nodes. + +For clusters configured with a file ratehr than dynamic config it's +advised to make sure that the configuration file is correct, although +it's not required. + +Once the configuration is set to `external_storage` it the object can be +managed as a chart: + +```shell +kubectl get chart -n kube-system k0s-addon-chart-openebs -o yaml +``` + +Once migrated, it's recommended to perform an upgrade to 3.9.0: + +```shell +kubectl patch chart -n kube-system k0s-addon-chart-openebs --patch '{"spec":{"version":"3.9.0"}}' --type=merge +``` + +Optionally, if it's prefered to have the OpenEBS configuration as part of the +cluster configuration, it may be added it to the configuration file or cluster +configuration object following the steps of installing it from scratch. As long +as the values in the file match with the ones in the chart object this will not +trigger any changes. + +## Usage + +Once installed The cluster will have two storage classes available for you to use: + +```shell +k0s kubectl get storageclass +``` + +```shell +NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +openebs-device openebs.io/local Delete WaitForFirstConsumer false 24s +openebs-hostpath openebs.io/local Delete WaitForFirstConsumer false 24s +``` + +The `openebs-hostpath` is the storage class that maps to the `/var/openebs/local` + +The `openebs-device` is not configured and could be configured by [manifest deployer](manifests.md) accordingly to the [OpenEBS documentation](https://docs.openebs.io/) + +### Example + +Use following manifests as an example of pod with mounted volume: + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nginx-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + storageClassName: openebs-hostpath + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + namespace: default + labels: + app: nginx +spec: + selector: + matchLabels: + app: nginx + strategy: + type: Recreate + template: + metadata: + labels: + app: nginx + spec: + containers: + - image: nginx + name: nginx + volumeMounts: + - name: persistent-storage + mountPath: /var/lib/nginx + volumes: + - name: persistent-storage + persistentVolumeClaim: + claimName: nginx-pvc +``` + +```shell +k0s kubectl apply -f nginx.yaml +``` + +```shell +persistentvolumeclaim/nginx-pvc created +deployment.apps/nginx created +bash-5.1# k0s kc get pods +NAME READY STATUS RESTARTS AGE +nginx-d95bcb7db-gzsdt 1/1 Running 0 30s +``` + +```shell +k0s kubectl get pv +``` + +```shell +NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE +pvc-9a7fae2d-eb03-42c3-aaa9-1a807d5df12f 5Gi RWO Delete Bound default/nginx-pvc openebs-hostpath 30s +``` diff --git a/docs/storage.md b/docs/storage.md index 907f8e2ed83a..5904005e63a6 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -2,13 +2,49 @@ k0s supports any volume provider that implements the [CSI specification](https://github.com/container-storage-interface/spec). For convenience, k0s comes bundled in with support for [OpenEBS local path provisioner](https://openebs.io/docs/concepts/localpv). -The choise of which CSI provider to use depends heavily on the use case and infrastructure you're running on and the use case you have. +The choice of which CSI provider to use depends heavily on the use case and infrastructure you're running on and the use case you have. -## Bundled OpenEBS storage +## CSI + +k0s supports a wide range of different storage options by utilizing Container Storage Interface (CSI). All Kubernetes storage solutions are supported and users can easily select the storage that fits best for their needs. + +When the storage solution implements Container Storage Interface (CSI), containers can communicate with the storage for creation and configuration of persistent volumes. This makes it easy to dynamically provision the requested volumes. It also expands the supported storage solutions from the previous generation, in-tree volume plugins. More information about the CSI concept is described on the [Kubernetes Blog](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/). + +![k0s storage](img/k0s_storage.png) + +### Installing 3rd party storage solutions + +Follow your storage driver's installation instructions. Note that the Kubelet installed by k0s uses a slightly different path for its working directory (`/varlib/k0s/kubelet` instead of `/var/lib/kubelet`). Consult the CSI driver's configuration documentation on how to customize this path. + +## Example storage solutions + +Different Kubernetes storage solutions are explained in the [official Kubernetes storage documentation](https://kubernetes.io/docs/concepts/storage/volumes/). All of them can be used with k0s. Here are some popular ones: + +- Rook-Ceph (Open Source) +- MinIO (Open Source) +- Gluster (Open Source) +- Longhorn (Open Source) +- Amazon EBS +- Google Persistent Disk +- Azure Disk +- Portworx + +If you are looking for a fault-tolerant storage with data replication, you can find a k0s tutorial for configuring Ceph storage with Rook [in here](examples/rook-ceph.md). + +## Bundled OpenEBS storage (deprecated) + +Bundled OpenEBS was deprecated in favor of running it [as a helm extension](./examples/openebs.md), +this documentation is maintained as a reference for existing installations. + +This was done for two reasons: -K0s comes out with bundled OpenEBS installation which can be enabled by using [configuration file](./configuration.md) +1. By installing it as a helm extenssion the users have more control and flexibility without adding complexity. +2. Allows users to choose the OpenEBS version independently of their k0s version. +3. Makes k0s configuration more consistent. -Use following configuration as an example: +For new installations or to migrate existing installations refer to the [OpenEBS extension page](./examples/openebs.md). + +OpenEBS extension is enabled by setting spec.extensions.storage.type to `openebs_local_storage`: ```yaml spec: @@ -101,30 +137,3 @@ k0s kubectl get pv NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE pvc-9a7fae2d-eb03-42c3-aaa9-1a807d5df12f 5Gi RWO Delete Bound default/nginx-pvc openebs-hostpath 30s ``` - -## CSI - -k0s supports a wide range of different storage options by utilizing Container Storage Interface (CSI). All Kubernetes storage solutions are supported and users can easily select the storage that fits best for their needs. - -When the storage solution implements Container Storage Interface (CSI), containers can communicate with the storage for creation and configuration of persistent volumes. This makes it easy to dynamically provision the requested volumes. It also expands the supported storage solutions from the previous generation, in-tree volume plugins. More information about the CSI concept is described on the [Kubernetes Blog](https://kubernetes.io/blog/2019/01/15/container-storage-interface-ga/). - -![k0s storage](img/k0s_storage.png) - -### Installing 3rd party storage solutions - -Follow your storage driver's installation instructions. Note that the Kubelet installed by k0s uses a slightly different path for its working directory (`/varlib/k0s/kubelet` instead of `/var/lib/kubelet`). Consult the CSI driver's configuration documentation on how to customize this path. - -## Example storage solutions - -Different Kubernetes storage solutions are explained in the [official Kubernetes storage documentation](https://kubernetes.io/docs/concepts/storage/volumes/). All of them can be used with k0s. Here are some popular ones: - -- Rook-Ceph (Open Source) -- MinIO (Open Source) -- Gluster (Open Source) -- Longhorn (Open Source) -- Amazon EBS -- Google Persistent Disk -- Azure Disk -- Portworx - -If you are looking for a fault-tolerant storage with data replication, you can find a k0s tutorial for configuring Ceph storage with Rook [in here](examples/rook-ceph.md). diff --git a/mkdocs.yml b/mkdocs.yml index 0871158b8427..0085b77f119c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -62,6 +62,7 @@ nav: - Ambassador API Gateway: examples/ambassador-ingress.md - Ceph Storage with Rook: examples/rook-ceph.md - GitOps with Flux: examples/gitops-flux.md + - OpenEBS storage: examples/openebs.md - Troubleshooting: - FAQ: FAQ.md - Dockershim deprecation: dockershim.md diff --git a/pkg/apis/k0s/v1beta1/extensions.go b/pkg/apis/k0s/v1beta1/extensions.go index fdf225a89b01..db5a7502bbe7 100644 --- a/pkg/apis/k0s/v1beta1/extensions.go +++ b/pkg/apis/k0s/v1beta1/extensions.go @@ -28,6 +28,8 @@ var _ Validateable = (*ClusterExtensions)(nil) // ClusterExtensions specifies cluster extensions type ClusterExtensions struct { + //+kubebuilder:deprecatedversion:warning="storage is deprecated and will be removed in 1.30. https://docs.k0sproject.io/stable/examples/openebs". + // Storage *StorageExtension `json:"storage"` Helm *HelmExtensions `json:"helm"` } diff --git a/pkg/component/controller/extensions_controller.go b/pkg/component/controller/extensions_controller.go index 6bf57603e36e..ca45c3037b54 100644 --- a/pkg/component/controller/extensions_controller.go +++ b/pkg/component/controller/extensions_controller.go @@ -82,15 +82,9 @@ func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0 ec.L.Info("Extensions reconciliation started") defer ec.L.Info("Extensions reconciliation finished") - helmSettings := clusterConfig.Spec.Extensions.Helm - var err error - switch clusterConfig.Spec.Extensions.Storage.Type { - case k0sAPI.OpenEBSLocal: - helmSettings, err = addOpenEBSHelmExtension(helmSettings, clusterConfig.Spec.Extensions.Storage) - if err != nil { - ec.L.WithError(err).Error("Can't add openebs helm extension") - } - default: + helmSettings, err := ec.configureStorage(clusterConfig) + if err != nil { + return fmt.Errorf("Cannot configure storage: %w", err) } if err := ec.reconcileHelmExtensions(helmSettings); err != nil { @@ -100,6 +94,24 @@ func (ec *ExtensionsController) Reconcile(ctx context.Context, clusterConfig *k0 return nil } +func (ec *ExtensionsController) configureStorage(clusterConfig *k0sAPI.ClusterConfig) (*k0sAPI.HelmExtensions, error) { + helmSettings := clusterConfig.Spec.Extensions.Helm + if clusterConfig.Spec.Extensions.Storage.Type != k0sAPI.OpenEBSLocal { + return helmSettings, nil + } + + for _, chart := range helmSettings.Charts { + if chart.ChartName == "openebs-internal/openebs" { + return nil, fmt.Errorf("openebs-internal/openebs is defined in spec.extensions.helm.charts and spec.extensions.storage.type is set to openebs_local_storage. https://docs.k0sproject.io/stable/examples/openebs") + } + } + helmSettings, err := addOpenEBSHelmExtension(helmSettings, clusterConfig.Spec.Extensions.Storage) + if err != nil { + return nil, fmt.Errorf("Can't add openebs helm extension") + } + return helmSettings, nil +} + func addOpenEBSHelmExtension(helmSpec *k0sAPI.HelmExtensions, storageExtension *k0sAPI.StorageExtension) (*k0sAPI.HelmExtensions, error) { openEBSValues := map[string]interface{}{ "localprovisioner": map[string]interface{}{ diff --git a/pkg/component/controller/extensions_controller_test.go b/pkg/component/controller/extensions_controller_test.go index 9027da4c579e..3a49df5ad86e 100644 --- a/pkg/component/controller/extensions_controller_test.go +++ b/pkg/component/controller/extensions_controller_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" + k0sAPI "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" "github.com/stretchr/testify/assert" ) @@ -128,9 +129,71 @@ func TestChartNeedsUpgrade(t *testing.T) { cr := new(ChartReconciler) for _, tc := range testCases { - t.Run("", func(t *testing.T) { + t.Run(tc.description, func(t *testing.T) { actual := cr.chartNeedsUpgrade(tc.chart) assert.Equal(t, tc.expected, actual) }) } } + +func addHelmExtension(config *k0sAPI.ClusterConfig) *k0sAPI.ClusterConfig { + config.Spec.Extensions.Storage.Type = k0sAPI.OpenEBSLocal + return config +} +func addStorageExtension(config *k0sAPI.ClusterConfig) *k0sAPI.ClusterConfig { + config.Spec.Extensions.Helm, _ = addOpenEBSHelmExtension(config.Spec.Extensions.Helm, config.Spec.Extensions.Storage) + return config +} + +func TestConfigureStorage(t *testing.T) { + var testCases = []struct { + description string + clusterConfig *k0sAPI.ClusterConfig + expectedErr bool + expectedOpenEBS bool + }{ + { + "no_openebs", + k0sAPI.DefaultClusterConfig(), + false, + false, + }, + { + "openebs_helm_extension", + addHelmExtension(k0sAPI.DefaultClusterConfig()), + false, + true, + }, + { + "openebs_storage_extension", + addStorageExtension(k0sAPI.DefaultClusterConfig()), + false, + true, + }, + { + "openebs_both", + addStorageExtension(addHelmExtension(k0sAPI.DefaultClusterConfig())), + false, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + ec := ExtensionsController{} + helmSettings, err := ec.configureStorage(tc.clusterConfig) + + if tc.expectedErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + if tc.expectedOpenEBS { + assert.Equal(t, "openebs", helmSettings.Charts[0].Name) + assert.Equal(t, 1, len(helmSettings.Charts)) + } + }) + } + +}