From 1c792c2bf56b8fa072e9baefc5ff145943bbcabe Mon Sep 17 00:00:00 2001 From: Himanshu Ahirwar <83774016+h1manshu98@users.noreply.github.com> Date: Mon, 25 Sep 2023 21:49:31 +0530 Subject: [PATCH] Feat/addon velero (#27) * feat: New helm addon Velero, version 5.0.2 * feat: irsa with ec2-volume and s3 permission to store and fetch backups * feat: use irsa instead of IAM-Keys for velero * feat: update velero script in basic example * fix: tf-lint bucket name issue * fix: readme github-action * terraform fmt * comment fixes --- _examples/basic/.tflint.hcl => .tflint.hcl | 0 README.md | 20 +- README.yaml | 112 ++++ _examples/basic/main.tf | 6 +- _examples/basic/variables.tf | 10 + .../complete/config/override-fluent-bit.yaml | 3 +- .../complete/config/override-kubeclarity.yaml | 21 +- .../complete/config/override-velero.yaml | 36 ++ _examples/complete/main.tf | 20 +- _examples/complete/outputs.tf | 12 + _examples/complete/variables.tf | 9 + addons/fluent-bit/main.tf | 4 +- addons/kubeclarity/main.tf | 10 - addons/velero/README.md | 22 + .../velero/config/velero_default-values.yaml | 578 ++++++++++++++++++ addons/velero/data.tf | 3 + addons/velero/locals.tf | 43 ++ addons/velero/main.tf | 80 +++ addons/velero/outputs.tf | 19 + addons/velero/variables.tf | 46 ++ addons/velero/versions.tf | 10 + main.tf | 13 + outputs.tf | 17 + override_values.tf | 44 ++ variables.tf | 25 + 25 files changed, 1132 insertions(+), 31 deletions(-) rename _examples/basic/.tflint.hcl => .tflint.hcl (100%) create mode 100644 README.yaml create mode 100644 _examples/complete/config/override-velero.yaml create mode 100644 addons/velero/README.md create mode 100644 addons/velero/config/velero_default-values.yaml create mode 100644 addons/velero/data.tf create mode 100644 addons/velero/locals.tf create mode 100644 addons/velero/main.tf create mode 100644 addons/velero/outputs.tf create mode 100644 addons/velero/variables.tf create mode 100644 addons/velero/versions.tf diff --git a/_examples/basic/.tflint.hcl b/.tflint.hcl similarity index 100% rename from _examples/basic/.tflint.hcl rename to .tflint.hcl diff --git a/README.md b/README.md index 37b8c52..e94da33 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ |istio_ingress|False | No | |kiali_server|False | No | |fluent_bit|False | No | +|new_relic|False | No | +|velero|False | No |
@@ -76,6 +78,8 @@ |ingress_nginx_helm_config | [click here](https://github.com/clouddrove/terraform-aws-eks-addons/blob/master/override_values.tf#L337-L380) | No | |kubeclarity_helm_config | [click here](https://github.com/clouddrove/terraform-aws-eks-addons/blob/master/override_values.tf#L389-L410) | No | |fluent_bit_helm_config | [click here](https://github.com/clouddrove/terraform-aws-eks-addons/blob/master/override_values.tf#L420-L509) | No | +|new_relic_helm_config | [click here](https://github.com/clouddrove/terraform-aws-eks-addons/blob/master/override_values.tf#L519-L536) | No | +|velero_helm_config | [click here](https://github.com/clouddrove/terraform-aws-eks-addons/blob/master/override_values.tf#L545-L580) | No |
@@ -96,6 +100,8 @@ | ingress_nginx_extra_configs | No | | kubeclarity_extra_configs | No | | fluent_bit_extra_configs | No | +| new_relic_extra_configs | No | +| velero_extra_configs | No | ## Outputs @@ -154,6 +160,14 @@ | fluent_bit_namespace | namespace where fluent-bit is deployed | | fluent_bit_chart_version | Chart version of fluent-bit addon's helmchart | | fluent_bit_repository | Repository URL of fluent-bit helmchart | +| new_relic_namespace | namespace where new-relic is deployed | +| new_relic_chart_version | Chart version of new-relic addon's helmchart | +| new_relic_repository | Repository URL of new-relic helmchart | +| velero_service_account | ServiceAccount name created by IRSA module for velero| +| velero_iam_policy | IAM Policy used to create IRSA | +| velero_namespace | namespace where velero is deployed | +| velero_chart_version | Chart version of velero addon's helmchart | +| velero_repository | Repository URL of velero helmchart | ## How to Use @@ -164,9 +178,9 @@ ```bash module "addons" { source = "clouddrove/eks-addons/aws" - version = "0.0.4" + version = "0.0.6" - depends_on = [module.eks.cluster_name] + depends_on = [module.eks] eks_cluster_name = module.eks.cluster_name # -- Enable Addons @@ -180,6 +194,8 @@ module "addons" { calico_tigera = false kubeclarity = true ingress_nginx = true + velero = true + new_relic = true # -- Addons with mandatory variable istio_ingress = true diff --git a/README.yaml b/README.yaml new file mode 100644 index 0000000..794e545 --- /dev/null +++ b/README.yaml @@ -0,0 +1,112 @@ +--- +# +# This is the canonical configuration for the `README.md` +# Run `make readme` to rebuild the `README.md` + +# Name of this project +name: Terraform AWS EKS ADDONS +# License of this project +license: "APACHE" + +# Canonical GitHub repo +github_repo: clouddrove/terraform-aws-eks-addons + +# Badges to display +badges: + - name: "Latest Release" + image: "https://img.shields.io/github/release/clouddrove/terraform-aws-eks-addons.svg" + url: "https://github.com/clouddrove/terraform-aws-eks-addons/releases/latest" + - name: "tfsec" + image: "https://github.com/clouddrove/terraform-aws-eks-addons/actions/workflows/tfsec.yml/badge.svg" + url: "https://github.com/clouddrove/terraform-aws-eks-addons/actions/workflows/tfsec.yml" + - name: "Licence" + image: "https://img.shields.io/badge/License-APACHE-blue.svg" + url: "LICENSE.md" + +prerequesties: + - name: Terraform 1.4.6 + url: https://learn.hashicorp.com/terraform/getting-started/install.html + +# description of this project +description: |- + A Terraform Addons module to customize & install widely used helmchart during or after creation of your AWS EKS cluster. +# extra content +include: + - "terraform.md" + +# How to use this project +# How to use this project +usage: |- + Here are some examples of how you can use this module in your inventory structure: + + ### addons basic example + ```hcl + module "addons" { + source = "clouddrove/eks-addons/aws" + version = "0.0.6" + + depends_on = [module.eks] + eks_cluster_name = module.eks.cluster_name + + # -- Enable Addons + aws_load_balancer_controller = true + aws_efs_csi_driver = true + calico_tigera = true + fluent_bit = true + + # -- Addons with mandatory manifest files + istio_ingress = true + istio_manifests = { + istio_ingress_manifest_file_path = ["./config/istio/ingress.yaml", "./config/istio/ingress-internal.yaml"] + istio_gateway_manifest_file_path = ["./config/istio/gateway.yaml"] + } + kiali_server = true + kiali_manifests = { + kiali_virtualservice_file_path = "./config/kiali/kiali_vs.yaml" + } + external_secrets = true + externalsecrets_manifests = { + secret_store_manifest_file_path = "./config/external-secret/secret-store.yaml" + external_secrets_manifest_file_path = "./config/external-secret/external-secret.yaml" + secret_manager_name = "external_secrets" + } + } + ``` + + ### addons complete example + ```hcl + module "addons" { + source = "clouddrove/eks-addons/aws" + version = "0.0.6" + + depends_on = [module.eks] + eks_cluster_name = module.eks.cluster_name + + # -- Enable Addons + aws_load_balancer_controller = true + aws_efs_csi_driver = true + calico_tigera = true + + # -- Addons with mandatory manifest files + istio_ingress = true + istio_manifests = { + istio_ingress_manifest_file_path = ["./config/istio/ingress.yaml", "./config/istio/ingress-internal.yaml"] + istio_gateway_manifest_file_path = ["./config/istio/gateway.yaml"] + } + + # -- Path of override-values.yaml file + aws_load_balancer_controller_helm_config = { values = [file("./config/override-aws-load-balancer-controller.yaml")] } + aws_efs_csi_driver_helm_config = { values = [file("./config/override-aws-efs-csi-driver.yaml")] } + calico_tigera_helm_config = { values = [file("./config/calico-tigera-values.yaml")] } + istio_ingress_helm_config = { values = [file("./config/istio/override-values.yaml")] } + + # -- Override Helm Release attributes + aws_load_balancer_controller_extra_configs = var.aws_load_balancer_controller_extra_configs + aws_efs_csi_driver_extra_configs = var.aws_efs_csi_driver_extra_configs + calico_tigera_extra_configs = var.calico_tigera_extra_configs + istio_ingress_extra_configs = var.istio_ingress_extra_configs + + # -- Custome IAM Policy Json Content or Json file path + cluster_autoscaler_iampolicy_json_content = file("./custom-iam-policies/cluster-autoscaler.json") + } + ``` \ No newline at end of file diff --git a/_examples/basic/main.tf b/_examples/basic/main.tf index 08fd3c3..bf6de50 100644 --- a/_examples/basic/main.tf +++ b/_examples/basic/main.tf @@ -156,7 +156,7 @@ resource "aws_iam_policy" "node_additional" { module "addons" { source = "../../" - depends_on = [module.eks.cluster_name] + depends_on = [module.eks] eks_cluster_name = module.eks.cluster_name # -- Enable Addons @@ -171,6 +171,7 @@ module "addons" { kubeclarity = true ingress_nginx = true fluent_bit = true + velero = true # -- Addons with mandatory variable istio_ingress = true @@ -179,4 +180,7 @@ module "addons" { kiali_manifests = var.kiali_manifests external_secrets = true externalsecrets_manifests = var.externalsecrets_manifests + + # -- Extra helm_release attributes + velero_extra_configs = var.velero_extra_configs } diff --git a/_examples/basic/variables.tf b/_examples/basic/variables.tf index 85c1432..4220099 100644 --- a/_examples/basic/variables.tf +++ b/_examples/basic/variables.tf @@ -36,4 +36,14 @@ variable "externalsecrets_manifests" { secret_manager_name = "external_secrets" } description = "yaml manifest file path to create ExternalSecret, SecretStore and custome SecretManger name" +} + +#------------ EXTRA CONFIGS ----------- +variable "velero_extra_configs" { + type = any + default = { + timeout = 300 + atomic = true + bucket_name = "velero-addons" + } } \ No newline at end of file diff --git a/_examples/complete/config/override-fluent-bit.yaml b/_examples/complete/config/override-fluent-bit.yaml index 533cbd0..bf2207e 100644 --- a/_examples/complete/config/override-fluent-bit.yaml +++ b/_examples/complete/config/override-fluent-bit.yaml @@ -81,4 +81,5 @@ config: log_group_name /aws/containerinsights/{{ .Values.eks_configs.cluster_name }}/application auto_create_group true extra_user_agent container-insights - log_stream_prefix eks- \ No newline at end of file + log_stream_prefix eks- + log_retention_days 7 \ No newline at end of file diff --git a/_examples/complete/config/override-kubeclarity.yaml b/_examples/complete/config/override-kubeclarity.yaml index 8c32d13..591a9ae 100644 --- a/_examples/complete/config/override-kubeclarity.yaml +++ b/_examples/complete/config/override-kubeclarity.yaml @@ -11,12 +11,17 @@ kubeclarity: podAnnotations: co.elastic.logs/enabled: "true" - -# Be careful when using ingress. As there is no authentication on Kubeclarity yet, your instance may be accessible. -# Make sure the ingress remains internal if you decide to enable it. - service: - type: LoadBalancer - port: 80 + # -- Application Load Balancer + ingress: + enabled: true + labels: {} annotations: - service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" - service.beta.kubernetes.io/aws-load-balancer-name: "kubeclarity" \ No newline at end of file + kubernetes.io/ingress.class: alb + alb.ingress.kubernetes.io/group.name: ingress + alb.ingress.kubernetes.io/load-balancer-name: kubeclarity-eks-alb + alb.ingress.kubernetes.io/target-type: ip + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}]' + + hosts: + - host: test.addons.com \ No newline at end of file diff --git a/_examples/complete/config/override-velero.yaml b/_examples/complete/config/override-velero.yaml new file mode 100644 index 0000000..cb8d398 --- /dev/null +++ b/_examples/complete/config/override-velero.yaml @@ -0,0 +1,36 @@ +initContainers: + - name: velero-plugin-for-aws + image: velero/velero-plugin-for-aws:v1.7.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /target + name: plugins + +## Parameters for the `default` BackupStorageLocation and VolumeSnapshotLocation, +## and additional server settings. +configuration: + backupStorageLocation: + - name: aws + default: "true" + provider: aws + + volumeSnapshotLocation: + - name: aws + provider: aws + config: + region: "us-east-1" + + +# Info about the secret to be used by the Velero deployment, which +# should contain credentials for the cloud provider IAM account you've +# set up for Velero. +credentials: + useSecret: false + secretContents: {} + + +# Whether to deploy the node-agent daemonset. +deployNodeAgent: true +nodeAgent: + podVolumePath: /var/lib/kubelet/pods + privileged: true \ No newline at end of file diff --git a/_examples/complete/main.tf b/_examples/complete/main.tf index b7c87bd..e630f79 100644 --- a/_examples/complete/main.tf +++ b/_examples/complete/main.tf @@ -149,7 +149,7 @@ resource "aws_iam_policy" "node_additional" { module "addons" { source = "../../" - depends_on = [module.eks.cluster_name] + depends_on = [module.eks] eks_cluster_name = module.eks.cluster_name # -- Enable Addons @@ -159,12 +159,14 @@ module "addons" { aws_node_termination_handler = true aws_efs_csi_driver = true aws_ebs_csi_driver = true - karpenter = true - calico_tigera = true - new_relic = false - kubeclarity = true - ingress_nginx = true - fluent_bit = true + # karpenter = false # -- Set to `false` or comment line to Uninstall Karpenter if installed using terraform. + calico_tigera = true + new_relic = true + kubeclarity = true + ingress_nginx = true + fluent_bit = true + velero = true + # -- Addons with mandatory variable istio_ingress = true istio_manifests = var.istio_manifests @@ -188,6 +190,7 @@ module "addons" { ingress_nginx_helm_config = { values = [file("./config/override-ingress-nginx.yaml")] } kubeclarity_helm_config = { values = [file("./config/override-kubeclarity.yaml")] } fluent_bit_helm_config = { values = [file("./config/override-fluent-bit.yaml")] } + velero_helm_config = { values = [file("./config/override-velero.yaml")] } new_relic_helm_config = { values = [file("./config/override-new-relic.yaml")] } # -- Override Helm Release attributes @@ -205,8 +208,9 @@ module "addons" { ingress_nginx_extra_configs = var.ingress_nginx_extra_configs kubeclarity_extra_configs = var.kubeclarity_extra_configs fluent_bit_extra_configs = var.fluent_bit_extra_configs + velero_extra_configs = var.velero_extra_configs new_relic_extra_configs = var.new_relic_extra_configs - # -- Custom IAM Policy Json Content or Json file path + # -- Custom IAM Policy Json for Addon's ServiceAccount cluster_autoscaler_iampolicy_json_content = file("./custom-iam-policies/cluster-autoscaler.json") } \ No newline at end of file diff --git a/_examples/complete/outputs.tf b/_examples/complete/outputs.tf index 7788e9b..c7f6694 100644 --- a/_examples/complete/outputs.tf +++ b/_examples/complete/outputs.tf @@ -7,4 +7,16 @@ output "cluster_name" { output "region" { value = local.region +} + +output "update_kubeconfig" { + value = "aws eks update-kubeconfig --name ${module.eks.cluster_name} --region ${local.region}" +} + +output "velero_post_installation" { + value = < +Velero is an open source tool to safely backup and restore, perform disaster recovery, and migrate Kubernetes cluster resources and persistent volumes. + +## Installation +Below terraform script shows how to use Velero Terraform Addon, A complete example is also given [here](https://github.com/clouddrove/terraform-helm-eks-addons/blob/master/_examples/complete/main.tf). + +```bash +module "addons" { + source = "clouddrove/eks-addons/aws" + version = "0.0.6" + + depends_on = [module.eks] + eks_cluster_name = module.eks.cluster_name + + velero = true + velero_helm_config = { values = ["${file("./path/to/override-velero.yaml")}"] } +} +``` + + diff --git a/addons/velero/config/velero_default-values.yaml b/addons/velero/config/velero_default-values.yaml new file mode 100644 index 0000000..463290d --- /dev/null +++ b/addons/velero/config/velero_default-values.yaml @@ -0,0 +1,578 @@ +## +## Configuration settings that directly affect the Velero deployment YAML. +## + +# Details of the container image to use in the Velero deployment & daemonset (if +# enabling node-agent). Required. +image: + repository: velero/velero + tag: v1.11.1 + # Digest value example: sha256:d238835e151cec91c6a811fe3a89a66d3231d9f64d09e5f3c49552672d271f38. + # If used, it will take precedence over the image.tag. + # digest: + pullPolicy: IfNotPresent + # One or more secrets to be used when pulling images + imagePullSecrets: [] + # - registrySecretName + +nameOverride: "" +fullnameOverride: "" + +# Annotations to add to the Velero deployment's. Optional. +# +# If you are using reloader use the following annotation with your VELERO_SECRET_NAME +annotations: {} +# secret.reloader.stakater.com/reload: "" + +# Labels to add to the Velero deployment's. Optional. +labels: {} + +# Annotations to add to the Velero deployment's pod template. Optional. +# +# If using kube2iam or kiam, use the following annotation with your AWS_ACCOUNT_ID +# and VELERO_ROLE_NAME filled in: +podAnnotations: {} + # iam.amazonaws.com/role: "arn:aws:iam:::role/" + +# Additional pod labels for Velero deployment's template. Optional +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ +podLabels: {} + +# Number of old history to retain to allow rollback (If not set, default Kubernetes value is set to 10) +# revisionHistoryLimit: 1 + +# Resource requests/limits to specify for the Velero deployment. +# https://velero.io/docs/v1.6/customize-installation/#customize-resource-requests-and-limits +resources: + requests: + cpu: 500m + memory: 128Mi + limits: + cpu: 1000m + memory: 512Mi + +# Configure the dnsPolicy of the Velero deployment +# See: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy +dnsPolicy: ClusterFirst + +# Init containers to add to the Velero deployment's pod spec. At least one plugin provider image is required. +# If the value is a string then it is evaluated as a template. +initContainers: + # - name: velero-plugin-for-csi + # image: velero/velero-plugin-for-csi:v0.5.0 + # imagePullPolicy: IfNotPresent + # volumeMounts: + # - mountPath: /target + # name: plugins + # - name: velero-plugin-for-aws + # image: velero/velero-plugin-for-aws:v1.7.0 + # imagePullPolicy: IfNotPresent + # volumeMounts: + # - mountPath: /target + # name: plugins + +# SecurityContext to use for the Velero deployment. Optional. +# Set fsGroup for `AWS IAM Roles for Service Accounts` +# see more informations at: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html +podSecurityContext: {} + # fsGroup: 1337 + +# Container Level Security Context for the 'velero' container of the Velero deployment. Optional. +# See: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container +containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: ["ALL"] + # add: [] + # readOnlyRootFilesystem: true + +# Container Lifecycle Hooks to use for the Velero deployment. Optional. +lifecycle: {} + +# Pod priority class name to use for the Velero deployment. Optional. +priorityClassName: "" + +# The number of seconds to allow for graceful termination of the pod. Optional. +terminationGracePeriodSeconds: 3600 + +# Liveness probe of the pod +livenessProbe: + httpGet: + path: /metrics + port: http-monitoring + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + +# Readiness probe of the pod +readinessProbe: + httpGet: + path: /metrics + port: http-monitoring + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + +# Tolerations to use for the Velero deployment. Optional. +tolerations: [] + +# Affinity to use for the Velero deployment. Optional. +affinity: {} + +# Node selector to use for the Velero deployment. Optional. +nodeSelector: {} + +# DNS configuration to use for the Velero deployment. Optional. +dnsConfig: {} + +# Extra volumes for the Velero deployment. Optional. +extraVolumes: [] + +# Extra volumeMounts for the Velero deployment. Optional. +extraVolumeMounts: [] + +# Extra K8s manifests to deploy +extraObjects: [] + # - apiVersion: secrets-store.csi.x-k8s.io/v1 + # kind: SecretProviderClass + # metadata: + # name: velero-secrets-store + # spec: + # provider: aws + # parameters: + # objects: | + # - objectName: "velero" + # objectType: "secretsmanager" + # jmesPath: + # - path: "access_key" + # objectAlias: "access_key" + # - path: "secret_key" + # objectAlias: "secret_key" + # secretObjects: + # - data: + # - key: access_key + # objectName: client-id + # - key: client-secret + # objectName: client-secret + # secretName: velero-secrets-store + # type: Opaque + +# Settings for Velero's prometheus metrics. Enabled by default. +metrics: + enabled: true + scrapeInterval: 30s + scrapeTimeout: 10s + + # service metdata if metrics are enabled + service: + annotations: {} + labels: {} + + # Pod annotations for Prometheus + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8085" + prometheus.io/path: "/metrics" + + serviceMonitor: + autodetect: true + enabled: false + annotations: {} + additionalLabels: {} + + # metrics.serviceMonitor.metricRelabelings Specify Metric Relabelings to add to the scrape endpoint + # ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + # metricRelabelings: [] + # metrics.serviceMonitor.relabelings [array] Prometheus relabeling rules + # relabelings: [] + # ServiceMonitor namespace. Default to Velero namespace. + # namespace: + # ServiceMonitor connection scheme. Defaults to HTTP. + # scheme: "" + # ServiceMonitor connection tlsConfig. Defaults to {}. + # tlsConfig: {} + nodeAgentPodMonitor: + autodetect: true + enabled: false + annotations: {} + additionalLabels: {} + # ServiceMonitor namespace. Default to Velero namespace. + # namespace: + # ServiceMonitor connection scheme. Defaults to HTTP. + # scheme: "" + # ServiceMonitor connection tlsConfig. Defaults to {}. + # tlsConfig: {} + + prometheusRule: + autodetect: true + enabled: false + # Additional labels to add to deployed PrometheusRule + additionalLabels: {} + # PrometheusRule namespace. Defaults to Velero namespace. + # namespace: "" + # Rules to be deployed + spec: [] + # - alert: VeleroBackupPartialFailures + # annotations: + # message: Velero backup {{ $labels.schedule }} has {{ $value | humanizePercentage }} partialy failed backups. + # expr: |- + # velero_backup_partial_failure_total{schedule!=""} / velero_backup_attempt_total{schedule!=""} > 0.25 + # for: 15m + # labels: + # severity: warning + # - alert: VeleroBackupFailures + # annotations: + # message: Velero backup {{ $labels.schedule }} has {{ $value | humanizePercentage }} failed backups. + # expr: |- + # velero_backup_failure_total{schedule!=""} / velero_backup_attempt_total{schedule!=""} > 0.25 + # for: 15m + # labels: + # severity: warning + +kubectl: + image: + repository: docker.io/bitnami/kubectl + # Digest value example: sha256:d238835e151cec91c6a811fe3a89a66d3231d9f64d09e5f3c49552672d271f38. + # If used, it will take precedence over the kubectl.image.tag. + # digest: + # kubectl image tag. If used, it will take precedence over the cluster Kubernetes version. + # tag: 1.16.15 + # Container Level Security Context for the 'kubectl' container of the crd jobs. Optional. + # See: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + # Resource requests/limits to specify for the upgrade/cleanup job. Optional + resources: {} + # Annotations to set for the upgrade/cleanup job. Optional. + annotations: {} + # Labels to set for the upgrade/cleanup job. Optional. + labels: {} + +# This job upgrades the CRDs. +upgradeCRDs: true + +# This job is meant primarily for cleaning up CRDs on CI systems. +# Using this on production systems, especially those that have multiple releases of Velero, will be destructive. +cleanUpCRDs: false + +## +## End of deployment-related settings. +## + + +## +## Parameters for the `default` BackupStorageLocation and VolumeSnapshotLocation, +## and additional server settings. +## +configuration: + # Parameters for the BackupStorageLocation(s). Configure multiple by adding other element(s) to the backupStorageLocation slice. + # See https://velero.io/docs/v1.6/api-types/backupstoragelocation/ + backupStorageLocation: + # name is the name of the backup storage location where backups should be stored. If a name is not provided, + # a backup storage location will be created with the name "default". Optional. + - name: + # provider is the name for the backup storage location provider. + provider: + # bucket is the name of the bucket to store backups in. Required. + bucket: + # caCert defines a base64 encoded CA bundle to use when verifying TLS connections to the provider. Optional. + caCert: + # prefix is the directory under which all Velero data should be stored within the bucket. Optional. + prefix: + # default indicates this location is the default backup storage location. Optional. + default: + # validationFrequency defines how frequently Velero should validate the object storage. Optional. + validationFrequency: + # accessMode determines if velero can write to this backup storage location. Optional. + # default to ReadWrite, ReadOnly is used during migrations and restores. + accessMode: ReadWrite + credential: + # name of the secret used by this backupStorageLocation. + name: + # name of key that contains the secret data to be used. + key: + # Additional provider-specific configuration. See link above + # for details of required/optional fields for your provider. + config: {} + # region: + # s3ForcePathStyle: + # s3Url: + # kmsKeyId: + # resourceGroup: + # The ID of the subscription containing the storage account, if different from the cluster’s subscription. (Azure only) + # subscriptionId: + # storageAccount: + # publicUrl: + # Name of the GCP service account to use for this backup storage location. Specify the + # service account here if you want to use workload identity instead of providing the key file.(GCP only) + # serviceAccount: + # Option to skip certificate validation or not if insecureSkipTLSVerify is set to be true, the client side should set the + # flag. For Velero client Command like velero backup describe, velero backup logs needs to add the flag --insecure-skip-tls-verify + # insecureSkipTLSVerify: + + # Parameters for the VolumeSnapshotLocation(s). Configure multiple by adding other element(s) to the volumeSnapshotLocation slice. + # See https://velero.io/docs/v1.6/api-types/volumesnapshotlocation/ + volumeSnapshotLocation: + # name is the name of the volume snapshot location where snapshots are being taken. Required. + - name: + # provider is the name for the volume snapshot provider. + provider: + credential: + # name of the secret used by this volumeSnapshotLocation. + name: + # name of key that contains the secret data to be used. + key: + # Additional provider-specific configuration. See link above + # for details of required/optional fields for your provider. + config: {} + # region: + # apiTimeout: + # resourceGroup: + # The ID of the subscription where volume snapshots should be stored, if different from the cluster’s subscription. If specified, also requires `configuration.volumeSnapshotLocation.config.resourceGroup`to be set. (Azure only) + # subscriptionId: + # incremental: + # snapshotLocation: + # project: + + # These are server-level settings passed as CLI flags to the `velero server` command. Velero + # uses default values if they're not passed in, so they only need to be explicitly specified + # here if using a non-default value. The `velero server` default values are shown in the + # comments below. + # -------------------- + # `velero server` default: restic + uploaderType: + # `velero server` default: 1m + backupSyncPeriod: + # `velero server` default: 4h + fsBackupTimeout: + # `velero server` default: 30 + clientBurst: + # `velero server` default: 500 + clientPageSize: + # `velero server` default: 20.0 + clientQPS: + # Name of the default backup storage location. Default: default + defaultBackupStorageLocation: + # How long to wait by default before backups can be garbage collected. Default: 72h + defaultBackupTTL: + # Name of the default volume snapshot location. + defaultVolumeSnapshotLocations: + # `velero server` default: empty + disableControllers: + # `velero server` default: 1h + garbageCollectionFrequency: + # Set log-format for Velero pod. Default: text. Other option: json. + logFormat: + # Set log-level for Velero pod. Default: info. Other options: debug, warning, error, fatal, panic. + logLevel: + # The address to expose prometheus metrics. Default: :8085 + metricsAddress: + # Directory containing Velero plugins. Default: /plugins + pluginDir: + # The address to expose the pprof profiler. Default: localhost:6060 + profilerAddress: + # `velero server` default: false + restoreOnlyMode: + # `velero server` default: customresourcedefinitions,namespaces,storageclasses,volumesnapshotclass.snapshot.storage.k8s.io,volumesnapshotcontents.snapshot.storage.k8s.io,volumesnapshots.snapshot.storage.k8s.io,persistentvolumes,persistentvolumeclaims,secrets,configmaps,serviceaccounts,limitranges,pods,replicasets.apps,clusterclasses.cluster.x-k8s.io,clusters.cluster.x-k8s.io,clusterresourcesets.addons.cluster.x-k8s.io + restoreResourcePriorities: + # `velero server` default: 1m + storeValidationFrequency: + # How long to wait on persistent volumes and namespaces to terminate during a restore before timing out. Default: 10m + terminatingResourceTimeout: + # Comma separated list of velero feature flags. default: empty + # features: EnableCSI + features: + # `velero server` default: velero + namespace: + + # additional key/value pairs to be used as environment variables such as "AWS_CLUSTER_NAME: 'yourcluster.domain.tld'" + extraEnvVars: {} + + # Set true for backup all pod volumes without having to apply annotation on the pod when used file system backup Default: false. + defaultVolumesToFsBackup: + + # How often repository maintain is run for repositories by default. + defaultRepoMaintainFrequency: + +## +## End of backup/snapshot location settings. +## + + +## +## Settings for additional Velero resources. +## + +rbac: + # Whether to create the Velero role and role binding to give all permissions to the namespace to Velero. + create: true + # Whether to create the cluster role binding to give administrator permissions to Velero + clusterAdministrator: true + # Name of the ClusterRole. + clusterAdministratorName: cluster-admin + +# Information about the Kubernetes service account Velero uses. +serviceAccount: + server: + create: true + name: + annotations: + labels: + +# Info about the secret to be used by the Velero deployment, which +# should contain credentials for the cloud provider IAM account you've +# set up for Velero. +credentials: + # Whether a secret should be used. Set to false if, for examples: + # - using kube2iam or kiam to provide AWS IAM credentials instead of providing the key file. (AWS only) + # - using workload identity instead of providing the key file. (GCP only) + useSecret: true + # Name of the secret to create if `useSecret` is true and `existingSecret` is empty + name: + # Name of a pre-existing secret (if any) in the Velero namespace + # that should be used to get IAM account credentials. Optional. + existingSecret: + # Data to be stored in the Velero secret, if `useSecret` is true and `existingSecret` is empty. + # As of the current Velero release, Velero only uses one secret key/value at a time. + # The key must be named `cloud`, and the value corresponds to the entire content of your IAM credentials file. + # Note that the format will be different for different providers, please check their documentation. + # Here is a list of documentation for plugins maintained by the Velero team: + # [AWS] https://github.com/vmware-tanzu/velero-plugin-for-aws/blob/main/README.md + # [GCP] https://github.com/vmware-tanzu/velero-plugin-for-gcp/blob/main/README.md + # [Azure] https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure/blob/main/README.md + secretContents: {} + # cloud: | + # [default] + # aws_access_key_id= + # aws_secret_access_key= + # additional key/value pairs to be used as environment variables such as "DIGITALOCEAN_TOKEN: ". Values will be stored in the secret. + extraEnvVars: {} + # Name of a pre-existing secret (if any) in the Velero namespace + # that will be used to load environment variables into velero and node-agent. + # Secret should be in format - https://kubernetes.io/docs/concepts/configuration/secret/#use-case-as-container-environment-variables + extraSecretRef: "" + +# Whether to create backupstoragelocation crd, if false => do not create a default backup location +backupsEnabled: true +# Whether to create volumesnapshotlocation crd, if false => disable snapshot feature +snapshotsEnabled: true + +# Whether to deploy the node-agent daemonset. +deployNodeAgent: false + +nodeAgent: + podVolumePath: /var/lib/kubelet/pods + privileged: false + # Pod priority class name to use for the node-agent daemonset. Optional. + priorityClassName: "" + # Resource requests/limits to specify for the node-agent daemonset deployment. Optional. + # https://velero.io/docs/v1.6/customize-installation/#customize-resource-requests-and-limits + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: 1000m + memory: 1024Mi + + # Tolerations to use for the node-agent daemonset. Optional. + tolerations: [] + + # Annotations to set for the node-agent daemonset. Optional. + annotations: {} + + # labels to set for the node-agent daemonset. Optional. + labels: {} + + # will map /scratch to emptyDir. Set to false and specify your own volume + # via extraVolumes and extraVolumeMounts that maps to /scratch + # if you don't want to use emptyDir. + useScratchEmptyDir: true + + # Extra volumes for the node-agent daemonset. Optional. + extraVolumes: [] + + # Extra volumeMounts for the node-agent daemonset. Optional. + extraVolumeMounts: [] + + # Key/value pairs to be used as environment variables for the node-agent daemonset. Optional. + extraEnvVars: {} + + # Configure the dnsPolicy of the node-agent daemonset + # See: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy + dnsPolicy: ClusterFirst + + # SecurityContext to use for the Velero deployment. Optional. + # Set fsGroup for `AWS IAM Roles for Service Accounts` + # see more informations at: https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html + podSecurityContext: + runAsUser: 0 + # fsGroup: 1337 + + # Container Level Security Context for the 'node-agent' container of the node-agent daemonset. Optional. + # See: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + containerSecurityContext: {} + + # Container Lifecycle Hooks to use for the node-agent daemonset. Optional. + lifecycle: {} + + # Node selector to use for the node-agent daemonset. Optional. + nodeSelector: {} + + # Affinity to use with node-agent daemonset. Optional. + affinity: {} + + # DNS configuration to use for the node-agent daemonset. Optional. + dnsConfig: {} + +# Backup schedules to create. +# Eg: +# schedules: +# mybackup: +# disabled: false +# labels: +# myenv: foo +# annotations: +# myenv: foo +# schedule: "0 0 * * *" +# useOwnerReferencesInBackup: false +# template: +# ttl: "240h" +# storageLocation: default +# includedNamespaces: +# - foo +schedules: {} + +# Velero ConfigMaps. +# Eg: +# configMaps: + # See: https://velero.io/docs/v1.11/file-system-backup/ +# fs-restore-action-config: +# labels: +# velero.io/plugin-config: "" +# velero.io/pod-volume-restore: RestoreItemAction +# data: +# image: velero/velero-restore-helper:v1.10.2 +# cpuRequest: 200m +# memRequest: 128Mi +# cpuLimit: 200m +# memLimit: 128Mi +# secCtx: | +# capabilities: +# drop: +# - ALL +# add: [] +# allowPrivilegeEscalation: false +# readOnlyRootFilesystem: true +# runAsUser: 1001 +# runAsGroup: 999 +configMaps: {} + +## +## End of additional Velero resource settings. +## diff --git a/addons/velero/data.tf b/addons/velero/data.tf new file mode 100644 index 0000000..79bc388 --- /dev/null +++ b/addons/velero/data.tf @@ -0,0 +1,3 @@ +data "aws_eks_cluster" "eks_cluster" { + name = var.eks_cluster_name +} \ No newline at end of file diff --git a/addons/velero/locals.tf b/addons/velero/locals.tf new file mode 100644 index 0000000..535715e --- /dev/null +++ b/addons/velero/locals.tf @@ -0,0 +1,43 @@ +locals { + name = "velero" + + default_helm_config = { + name = try(var.velero_extra_configs.name, local.name) + chart = try(var.velero_extra_configs.chart, local.name) + repository = try(var.velero_extra_configs.repository, "https://vmware-tanzu.github.io/helm-charts") + version = try(var.velero_extra_configs.version, "5.0.2") + namespace = try(var.velero_extra_configs.namespace, "velero") + create_namespace = try(var.velero_extra_configs.create_namespace, true) + description = "Velero helm Chart deployment configuration" + timeout = try(var.velero_extra_configs.timeout, "600") + lint = try(var.velero_extra_configs.lint, "false") + repository_key_file = try(var.velero_extra_configs.repository_key_file, "") + repository_cert_file = try(var.velero_extra_configs.repository_cert_file, "") + repository_username = try(var.velero_extra_configs.repository_password, "") + repository_password = try(var.velero_extra_configs.repository_password, "") + verify = try(var.velero_extra_configs.verify, "false") + keyring = try(var.velero_extra_configs.keyring, "") + disable_webhooks = try(var.velero_extra_configs.disable_webhooks, "false") + reuse_values = try(var.velero_extra_configs.reuse_values, "false") + reset_values = try(var.velero_extra_configs.reset_values, "false") + force_update = try(var.velero_extra_configs.force_update, "false") + recreate_pods = try(var.velero_extra_configs.recreate_pods, "false") + cleanup_on_fail = try(var.velero_extra_configs.cleanup_on_fail, "false") + max_history = try(var.velero_extra_configs.max_history, "0") + atomic = try(var.velero_extra_configs.atomic, "false") + skip_crds = try(var.velero_extra_configs.skip_crds, "false") + render_subchart_notes = try(var.velero_extra_configs.render_subchart_notes, "true") + disable_openapi_validation = try(var.velero_extra_configs.disable_openapi_validation, "false") + wait = try(var.velero_extra_configs.wait, "true") + wait_for_jobs = try(var.velero_extra_configs.wait_for_jobs, "false") + dependency_update = try(var.velero_extra_configs.dependency_update, "false") + replace = try(var.velero_extra_configs.replace, "false") + } + + velero_extra_configs = var.velero_extra_configs + + helm_config = merge( + local.default_helm_config, + var.helm_config, + ) +} diff --git a/addons/velero/main.tf b/addons/velero/main.tf new file mode 100644 index 0000000..e76c47f --- /dev/null +++ b/addons/velero/main.tf @@ -0,0 +1,80 @@ +module "helm_addon" { + source = "../helm" + + manage_via_gitops = var.manage_via_gitops + helm_config = local.helm_config + addon_context = var.addon_context + + set_values = [ + { + name = "serviceAccount.server.create" + value = false + }, + { + name = "serviceAccount.server.name" + value = "${local.name}-sa" + }, + { + name = "configuration.backupStorageLocation[0].bucket" + value = try(var.velero_extra_configs.bucket_name, "velero-addons") + } + ] + + # -- IRSA Configurations + irsa_config = { + create_kubernetes_namespace = local.default_helm_config.create_namespace + create_kubernetes_service_account = true + kubernetes_namespace = local.default_helm_config.namespace + kubernetes_service_account = "${local.name}-sa" + irsa_iam_policies = [aws_iam_policy.policy.arn] + irsa_iam_role_name = "${local.name}-${var.eks_cluster_name}-IAM-Role" + eks_oidc_provider_arn = replace(data.aws_eks_cluster.eks_cluster.identity[0].oidc[0].issuer, "https://", "") + account_id = var.account_id + } +} +resource "aws_iam_policy" "policy" { + name = "${local.name}-${var.eks_cluster_name}-IAM-Policy" + path = "/" + description = "IAM Policy used by ${local.name}-${var.eks_cluster_name} IAM Role" + policy = var.iampolicy_json_content != null ? var.iampolicy_json_content : <<-EOT +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeVolumes", + "ec2:DescribeSnapshots", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:CreateSnapshot", + "ec2:DeleteSnapshot" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:DeleteObject", + "s3:PutObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::${try(var.velero_extra_configs.bucket_name, "velero-addons-${var.eks_cluster_name}")}/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${try(var.velero_extra_configs.bucket_name, "velero-addons-${var.eks_cluster_name}")}" + ] + } + ] +} + EOT +} \ No newline at end of file diff --git a/addons/velero/outputs.tf b/addons/velero/outputs.tf new file mode 100644 index 0000000..8f744a2 --- /dev/null +++ b/addons/velero/outputs.tf @@ -0,0 +1,19 @@ +output "service_account" { + value = "${local.name}-sa" +} + +output "iam_policy" { + value = "${local.name}-${var.eks_cluster_name}-IAM-Policy" +} + +output "namespace" { + value = local.default_helm_config.namespace +} + +output "chart_version" { + value = local.default_helm_config.version +} + +output "repository" { + value = local.default_helm_config.repository +} \ No newline at end of file diff --git a/addons/velero/variables.tf b/addons/velero/variables.tf new file mode 100644 index 0000000..275e5de --- /dev/null +++ b/addons/velero/variables.tf @@ -0,0 +1,46 @@ +variable "helm_config" { + description = "Helm provider config for Velero" + type = any + default = {} +} + +variable "manage_via_gitops" { + description = "Determines if the add-on should be managed via GitOps" + type = bool + default = false +} + +variable "addon_context" { + description = "Input configuration for the addon" + type = object({ + aws_caller_identity_account_id = string + aws_caller_identity_arn = string + aws_eks_cluster_endpoint = string + aws_partition_id = string + aws_region_name = string + eks_cluster_id = string + eks_oidc_issuer_url = string + eks_oidc_provider_arn = string + tags = map(string) + }) +} + +variable "eks_cluster_name" { + type = string + default = "" +} +variable "account_id" { + type = string + default = "" +} + +variable "velero_extra_configs" { + description = "Override attributes of helm_release terraform resource" + type = any +} + +variable "iampolicy_json_content" { + description = "Custom IAM Policy for Velero IRSA" + type = string + default = null +} \ No newline at end of file diff --git a/addons/velero/versions.tf b/addons/velero/versions.tf new file mode 100644 index 0000000..55fba73 --- /dev/null +++ b/addons/velero/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + } +} diff --git a/main.tf b/main.tf index dbbca2b..691b8b4 100644 --- a/main.tf +++ b/main.tf @@ -149,6 +149,7 @@ module "fluent_bit" { fluent_bit_extra_configs = var.fluent_bit_extra_configs iampolicy_json_content = var.fluent_bit_iampolicy_json_content } + module "new_relic" { count = var.new_relic ? 1 : 0 source = "./addons/nri-bundle" @@ -157,4 +158,16 @@ module "new_relic" { addon_context = local.addon_context eks_cluster_name = data.aws_eks_cluster.eks_cluster.name new_relic_extra_configs = var.new_relic_extra_configs +} + +module "velero" { + count = var.velero ? 1 : 0 + source = "./addons/velero" + helm_config = var.velero_helm_config != null ? var.velero_helm_config : { values = [local_file.velero_helm_config[count.index].content] } + manage_via_gitops = var.manage_via_gitops + addon_context = local.addon_context + eks_cluster_name = data.aws_eks_cluster.eks_cluster.name + account_id = data.aws_caller_identity.current.account_id + velero_extra_configs = var.velero_extra_configs + iampolicy_json_content = var.velero_iampolicy_json_content } \ No newline at end of file diff --git a/outputs.tf b/outputs.tf index c04b1b3..6170cf7 100644 --- a/outputs.tf +++ b/outputs.tf @@ -192,4 +192,21 @@ output "new_relic_chart_version" { } output "new_relic_repository" { value = module.new_relic[*].repository +} + +#----------- VELERO ---------------- +output "velero_service_account" { + value = module.velero[*].service_account +} +output "velero_iam_policy" { + value = module.velero[*].iam_policy +} +output "velero_namespace" { + value = module.velero[*].namespace +} +output "velero_chart_version" { + value = module.velero[*].chart_version +} +output "velero_repository" { + value = module.velero[*].repository } \ No newline at end of file diff --git a/override_values.tf b/override_values.tf index 0554d34..2bb1f92 100644 --- a/override_values.tf +++ b/override_values.tf @@ -537,3 +537,47 @@ global: EOT filename = "${path.module}/override_vales/new_relic.yaml" } + +#-----------VELERO ----------------------- +resource "local_file" "velero_helm_config" { + count = var.velero && (var.velero_helm_config == null) ? 1 : 0 + content = <