Refactoring End2End tests using K3s (#845)

* Update: refactoring kube-prometheus-stack cluster definition Signed-off-by: Nicolas Lamirault <[email protected]> * Update: refactoring Thanos cluster definition Signed-off-by: Nicolas Lamirault <[email protected]> * Add: CRD and Monitoring for Flux into CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: Grafana component Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: remove nfs dependency Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: k3s action debug Signed-off-by: Nicolas Lamirault <[email protected]> * Update: do not enable traefik on k3s cicd Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: Healthcheck namespace for Thanos Signed-off-by: Nicolas Lamirault <[email protected]> * Add: wait for Grafana on CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: storage class name on k3s Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: Grafana deployment on k3s Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: Grafana deployment on k3s Signed-off-by: Nicolas Lamirault <[email protected]> * Add: ChaosMesh on k3s for CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: ChaosMesh on k3s Signed-off-by: Nicolas Lamirault <[email protected]> * Fix: do not use ARM on CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: check ChaosMesh kustomization on CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: Litmus Chaos on K3s CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: LitmusChaos on K3s CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: Ingress Controller on k3s CICD Signed-off-by: Nicolas Lamirault <[email protected]> * Add: Kyverno on K3s CICD Signed-off-by: Nicolas Lamirault <[email protected]>
portefaix · Aug 17, 2021 · c8dce61 · c8dce61
1 parent 8c47d5a
commit c8dce61
Show file tree

Hide file tree

Showing 29 changed files with 984 additions and 70 deletions.
diff --git a/.github/workflows/flux-e2e.yml b/.github/workflows/flux-e2e.yml
@@ -55,27 +55,12 @@ jobs:
         CURRENT_BRANCH: ${{ github.head_ref }}
       if: github.event_name == 'pull_request'
 
-    # - name: Setup k3s
-    #   uses: debianmaster/[email protected]
-    #   id: k3s
-    #   with:
-    #     version: ${{ matrix.k8s_version }}
-
-    # - name: Check nodes on k3s
-    #   run: |
-    #     # To generate kubeconfigs for specific service accounts, current user must have read rights to the cluster-admin kubeconfig folder
-    #     sudo chown $(id -u):$(id -g) /tmp/output/
-    #     # export CLUSTER_ADMIN_KUBECONFIG=$KUBECONFIG
-
-    #     kubectl get nodes
-    #     kubectl get pods -A
-    #     sleep 20
-
     - name: Start k8s locally
       uses: jupyterhub/action-k3s-helm@v1
       with:
         k3s-version: ${{ matrix.k8s_version }}
         helm-version: v3.4.2 # releases:  https://github.com/helm/helm/tags
+        traefik-enabled: false
 
     - name: Verify function of k8s, kubectl, and helm
       run: |
@@ -122,11 +107,22 @@ jobs:
 
     - name: Verify reconciliation
       run: |
-        echo "Flux System"
+        echo "--> Pods"
+        kubectl get pods -A
+        echo "--> Flux System"
         kubectl -n flux-system wait kustomization/flux-system-charts --for=condition=ready --timeout=5m
-        echo "Monitoring"
+        echo "--> Monitoring"
         kubectl -n flux-system wait kustomization/kube-prometheus-stack --for=condition=ready --timeout=5m
         kubectl -n flux-system wait kustomization/thanos --for=condition=ready --timeout=5m
+        kubectl -n flux-system wait kustomization/grafana --for=condition=ready --timeout=5m
+        echo "--> Chaos"
+        kubectl -n flux-system wait kustomization/chaos-mesh --for=condition=ready --timeout=5m
+        kubectl -n flux-system wait kustomization/litmus-chaos --for=condition=ready --timeout=5m
+        echo "--> Ingress Controllers"
+        kubectl -n flux-system wait kustomization/ingress-controllers --for=condition=ready --timeout=5m
+        echo "--> Policies"
+        kubectl -n flux-system wait kustomization/kyverno --for=condition=ready --timeout=5m
+        kubectl -n flux-system wait kustomization/policy-reporter --for=condition=ready --timeout=5m
 
     - name: Debug failure
       if: failure()

diff --git a/clusters/k3s/cicd/chaos-mesh.yaml b/clusters/k3s/cicd/chaos-mesh.yaml
@@ -0,0 +1,44 @@
+# Copyright (C) 2021 Nicolas Lamirault <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: chaos-mesh
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  dependsOn:
+    # For ServiceMonitor CRD
+    - name: kube-prometheus-stack-crd
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/overlays/k3s/cicd/chaos-mesh/chaos-mesh
+  prune: true
+  validation: client
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: chaos-controller-manager
+      namespace: chaos-mesh
+    - apiVersion: apps/v1
+      kind: DaemonSet
+      name: chaos-daemon
+      namespace: chaos-mesh
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: chaos-dashboard
+      namespace: chaos-mesh
diff --git a/clusters/k3s/cicd/crds.yaml b/clusters/k3s/cicd/crds.yaml
@@ -0,0 +1,28 @@
+# Copyright (C) 2021 Nicolas Lamirault <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: kube-prometheus-stack-crd-source
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  path: ./kubernetes/overlays/k3s/cicd/crds/kube-prometheus-stack/
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  validation: client
diff --git a/clusters/k3s/cicd/flux-monitoring.yaml b/clusters/k3s/cicd/flux-monitoring.yaml
@@ -0,0 +1,31 @@
+# Copyright (C) 2021 Nicolas Lamirault <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: flux-monitoring
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  dependsOn:
+    # For PodMonitor CRD
+    - name: kube-prometheus-stack-crd
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/overlays/k3s/cicd/flux-system/monitoring
+  prune: true
+  validation: client
diff --git a/clusters/k3s/cicd/grafana.yaml b/clusters/k3s/cicd/grafana.yaml
@@ -0,0 +1,39 @@
+# Copyright (C) 2021 Nicolas Lamirault <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  decryption:
+    provider: sops
+    secretRef:
+      name: sops-gpg
+  interval: 10m0s
+  dependsOn:
+    - name: kube-prometheus-stack-crd
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/overlays/k3s/cicd/monitoring/grafana
+  prune: true
+  validation: client
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: StatefulSet
+      name: grafana
+      namespace: monitoring
diff --git a/clusters/k3s/cicd/ingress-nginx.yaml b/clusters/k3s/cicd/ingress-nginx.yaml
@@ -0,0 +1,35 @@
+# Copyright (C) 2021 Nicolas Lamirault <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: ingress-controllers
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  dependsOn:
+    - name: kube-prometheus-stack-crd
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./kubernetes/overlays/k3s/cicd/ingress-controllers/ingress-nginx
+  prune: true
+  validation: client
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: ingress-nginx-controller
+      namespace: ingress-controllers
diff --git a/clusters/k3s/cicd/kube-prometheus-stack-crd.yaml b/clusters/k3s/cicd/kube-prometheus-stack-crd.yaml
@@ -0,0 +1,54 @@
+# Copyright (C) 2021 Nicolas Lamirault <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
+kind: Kustomization
+metadata:
+  name: kube-prometheus-stack-crd
+  namespace: flux-system
+spec:
+  interval: 5m0s
+  dependsOn:
+    - name: kube-prometheus-stack-crd-source
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: kube-prometheus-stack-crd
+  validation: client
+  healthChecks:
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: alertmanagerconfigs.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: alertmanagers.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: podmonitors.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: probes.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: prometheuses.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: prometheusrules.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: servicemonitors.monitoring.coreos.com
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    name: thanosrulers.monitoring.coreos.com
diff --git a/clusters/k3s/cicd/kube-prometheus-stack.yaml b/clusters/k3s/cicd/kube-prometheus-stack.yaml
@@ -12,46 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
-kind: Kustomization
-metadata:
-  name: kube-prometheus-stack-crd
-  namespace: flux-system
-spec:
-  interval: 5m0s
-  dependsOn:
-    - name: kube-prometheus-stack-crd-source
-  prune: false
-  sourceRef:
-    kind: GitRepository
-    name: kube-prometheus-stack-crd
-  validation: client
-  healthChecks:
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: alertmanagerconfigs.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: alertmanagers.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: podmonitors.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: probes.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: prometheuses.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: prometheusrules.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: servicemonitors.monitoring.coreos.com
-  - apiVersion: apiextensions.k8s.io/v1
-    kind: CustomResourceDefinition
-    name: thanosrulers.monitoring.coreos.com
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
 kind: Kustomization
@@ -64,17 +24,32 @@ spec:
     secretRef:
       name: sops-gpg
   interval: 10m0s
-  # dependsOn:
-    # - name: sealed-secrets
-    # - name: flux-sops
+  dependsOn:
+    - name: kube-prometheus-stack-crd
   sourceRef:
     kind: GitRepository
     name: flux-system
   path: ./kubernetes/overlays/k3s/cicd/monitoring/kube-prometheus-stack
   prune: true
   validation: client
   healthChecks:
-    - apiVersion: helm.toolkit.fluxcd.io/v1beta1
-      kind: HelmRelease
-      name: kube-prometheus-stack
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: kube-prometheus-stack-operator
+      namespace: monitoring
+    - apiVersion: apps/v1
+      kind: StatefulSet
+      name: prometheus-kube-prometheus-stack-prometheus
+      namespace: monitoring
+    - apiVersion: apps/v1
+      kind: StatefulSet
+      name: alertmanager-kube-prometheus-stack-alertmanager
+      namespace: monitoring
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: kube-prometheus-stack-kube-state-metrics
+      namespace: monitoring
+    - apiVersion: apps/v1
+      kind: DaemonSet
+      name: kube-prometheus-stack-prometheus-node-exporter
       namespace: monitoring