diff --git a/charts/kyverno/Chart.yaml b/charts/kyverno/Chart.yaml index e25b6ee79010..60d2a6377a37 100644 --- a/charts/kyverno/Chart.yaml +++ b/charts/kyverno/Chart.yaml @@ -36,3 +36,9 @@ annotations: description: allow pod labels for cleanup jobs - kind: added description: allow nodeSelector for cleanup jobs + - kind: added + description: allow nodeSelector, tolerations and affinity settings for webhooksCleanup + - kind: added + description: allow affinity settings for cleanup jobs + - kind: added + description: Add helper to handle the labels for cleanup jobs, add component label diff --git a/charts/kyverno/README.md b/charts/kyverno/README.md index d35f3859a227..299639d7fc28 100644 --- a/charts/kyverno/README.md +++ b/charts/kyverno/README.md @@ -609,6 +609,11 @@ The chart values are organised per component. | webhooksCleanup.enabled | bool | `true` | Create a helm pre-delete hook to cleanup webhooks. | | webhooksCleanup.image | string | `"bitnami/kubectl:latest"` | `kubectl` image to run commands for deleting webhooks. | | webhooksCleanup.imagePullSecrets | list | `[]` | Image pull secrets | +| webhooksCleanup.nodeSelector | object | `{}` | Node labels for pod assignment | +| webhooksCleanup.tolerations | list | `[]` | List of node taints to tolerate | +| webhooksCleanup.podAntiAffinity | object | `{}` | Pod anti affinity constraints. | +| webhooksCleanup.podAffinity | object | `{}` | Pod affinity constraints. | +| webhooksCleanup.nodeAffinity | object | `{}` | Node affinity constraints. | ### Test @@ -648,6 +653,9 @@ The chart values are organised per component. | cleanupJobs.admissionReports.nodeSelector | object | `{}` | Node labels for pod assignment | | cleanupJobs.admissionReports.podAnnotations | object | `{}` | Pod Annotations | | cleanupJobs.admissionReports.podLabels | object | `{}` | Pod labels | +| cleanupJobs.admissionReports.podAntiAffinity | object | `{}` | Pod anti affinity constraints. | +| cleanupJobs.admissionReports.podAffinity | object | `{}` | Pod affinity constraints. | +| cleanupJobs.admissionReports.nodeAffinity | object | `{}` | Node affinity constraints. | | cleanupJobs.clusterAdmissionReports.enabled | bool | `true` | Enable cleanup cronjob | | cleanupJobs.clusterAdmissionReports.image.registry | string | `nil` | Image registry | | cleanupJobs.clusterAdmissionReports.image.repository | string | `"bitnami/kubectl"` | Image repository | @@ -664,6 +672,9 @@ The chart values are organised per component. | cleanupJobs.clusterAdmissionReports.nodeSelector | object | `{}` | Node labels for pod assignment | | cleanupJobs.clusterAdmissionReports.podAnnotations | object | `{}` | Pod Annotations | | cleanupJobs.clusterAdmissionReports.podLabels | object | `{}` | Pod Labels | +| cleanupJobs.clusterAdmissionReports.podAntiAffinity | object | `{}` | Pod anti affinity constraints. | +| cleanupJobs.clusterAdmissionReports.podAffinity | object | `{}` | Pod affinity constraints. | +| cleanupJobs.clusterAdmissionReports.nodeAffinity | object | `{}` | Node affinity constraints. | ### Other diff --git a/charts/kyverno/ci/cleanupJobs-values.yaml b/charts/kyverno/ci/cleanupJobs-values.yaml index 61e7e170250e..93c04d945cde 100644 --- a/charts/kyverno/ci/cleanupJobs-values.yaml +++ b/charts/kyverno/ci/cleanupJobs-values.yaml @@ -2,6 +2,28 @@ cleanupJobs: admissionReports: nodeSelector: kubernetes.io/os: linux + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - cleanup + topologyKey: kubernetes.io/hostname clusterAdmissionReports: nodeSelector: kubernetes.io/os: linux + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - cleanup + topologyKey: kubernetes.io/hostname diff --git a/charts/kyverno/ci/hooks-values.yaml b/charts/kyverno/ci/hooks-values.yaml index 399110f1b046..d0d98ab277ac 100644 --- a/charts/kyverno/ci/hooks-values.yaml +++ b/charts/kyverno/ci/hooks-values.yaml @@ -1,3 +1,16 @@ --- webhooksCleanup: enable: true + nodeSelector: + kubernetes.io/os: linux + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: + - hooks + topologyKey: kubernetes.io/hostname diff --git a/charts/kyverno/templates/cleanup/_helpers.tpl b/charts/kyverno/templates/cleanup/_helpers.tpl new file mode 100644 index 000000000000..a1b70cb3392a --- /dev/null +++ b/charts/kyverno/templates/cleanup/_helpers.tpl @@ -0,0 +1,9 @@ +{{/* vim: set filetype=mustache: */}} + +{{- define "kyverno.cleanup.labels" -}} +{{- template "kyverno.labels.merge" (list + (include "kyverno.labels.common" .) + (include "kyverno.matchLabels.common" .) + (include "kyverno.labels.component" "cleanup") +) -}} +{{- end -}} diff --git a/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml b/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml index cbe4a56c7c78..1a39820b4d10 100644 --- a/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml @@ -5,7 +5,7 @@ metadata: name: {{ template "kyverno.name" . }}-cleanup-admission-reports namespace: {{ template "kyverno.namespace" . }} labels: - {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }} + {{- include "kyverno.cleanup.labels" . | nindent 4 }} spec: schedule: {{ .Values.cleanupJobs.admissionReports.schedule | quote }} concurrencyPolicy: Forbid @@ -65,4 +65,19 @@ spec: nodeSelector: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} + {{- if or .Values.cleanupJobs.admissionReports.podAntiAffinity .Values.cleanupJobs.admissionReports.podAffinity .Values.cleanupJobs.admissionReports.nodeAffinity }} + affinity: + {{- with .Values.cleanupJobs.admissionReports.podAntiAffinity }} + podAntiAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.admissionReports.podAffinity }} + podAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.admissionReports.nodeAffinity }} + nodeAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- end }} {{- end -}} diff --git a/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml b/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml index 52c1ce7234c6..f67d8b0a7fbe 100644 --- a/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml @@ -5,7 +5,7 @@ metadata: name: {{ template "kyverno.name" . }}-cleanup-cluster-admission-reports namespace: {{ template "kyverno.namespace" . }} labels: - {{- include "kyverno.labels.merge" (list (include "kyverno.labels.common" .) (include "kyverno.matchLabels.common" .)) | nindent 4 }} + {{- include "kyverno.cleanup.labels" . | nindent 4 }} spec: schedule: {{ .Values.cleanupJobs.clusterAdmissionReports.schedule | quote }} concurrencyPolicy: Forbid @@ -65,4 +65,19 @@ spec: nodeSelector: {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} + {{- if or .Values.cleanupJobs.clusterAdmissionReports.podAntiAffinity .Values.cleanupJobs.clusterAdmissionReports.podAffinity .Values.cleanupJobs.clusterAdmissionReports.nodeAffinity }} + affinity: + {{- with .Values.cleanupJobs.clusterAdmissionReports.podAntiAffinity }} + podAntiAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.clusterAdmissionReports.podAffinity }} + podAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- with .Values.cleanupJobs.clusterAdmissionReports.nodeAffinity }} + nodeAffinity: + {{- tpl (toYaml .) $ | nindent 14 }} + {{- end }} + {{- end }} {{- end -}} diff --git a/charts/kyverno/templates/hooks/pre-delete.yaml b/charts/kyverno/templates/hooks/pre-delete.yaml index 522c867b6125..7479bc399ec5 100644 --- a/charts/kyverno/templates/hooks/pre-delete.yaml +++ b/charts/kyverno/templates/hooks/pre-delete.yaml @@ -31,5 +31,28 @@ spec: sleep 30 kubectl delete validatingwebhookconfiguration -l webhook.kyverno.io/managed-by=kyverno kubectl delete mutatingwebhookconfiguration -l webhook.kyverno.io/managed-by=kyverno + {{- with .Values.webhooksCleanup.tolerations }} + tolerations: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .Values.webhooksCleanup.nodeSelector }} + nodeSelector: + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- if or .Values.webhooksCleanup.podAntiAffinity .Values.webhooksCleanup.podAffinity .Values.webhooksCleanup.nodeAffinity }} + affinity: + {{- with .Values.webhooksCleanup.podAntiAffinity }} + podAntiAffinity: + {{- tpl (toYaml .) $ | nindent 10 }} + {{- end }} + {{- with .Values.webhooksCleanup.podAffinity }} + podAffinity: + {{- tpl (toYaml .) $ | nindent 10 }} + {{- end }} + {{- with .Values.webhooksCleanup.nodeAffinity }} + nodeAffinity: + {{- tpl (toYaml .) $ | nindent 10 }} + {{- end }} + {{- end }} {{- end -}} {{- end -}} diff --git a/charts/kyverno/values.yaml b/charts/kyverno/values.yaml index 4e084a1f70c1..1235554e2a77 100644 --- a/charts/kyverno/values.yaml +++ b/charts/kyverno/values.yaml @@ -308,6 +308,21 @@ webhooksCleanup: # -- Image pull secrets imagePullSecrets: [] + # -- Node labels for pod assignment + nodeSelector: {} + + # -- List of node taints to tolerate + tolerations: [] + + # -- Pod anti affinity constraints. + podAntiAffinity: {} + + # -- Pod affinity constraints. + podAffinity: {} + + # -- Node affinity constraints. + nodeAffinity: {} + grafana: # -- Enable grafana dashboard creation. enabled: false @@ -454,6 +469,15 @@ cleanupJobs: # -- Pod labels podLabels: {} + # -- Pod anti affinity constraints. + podAntiAffinity: {} + + # -- Pod affinity constraints. + podAffinity: {} + + # -- Node affinity constraints. + nodeAffinity: {} + clusterAdmissionReports: # -- Enable cleanup cronjob @@ -516,6 +540,15 @@ cleanupJobs: # -- Pod Labels podLabels: {} + # -- Pod anti affinity constraints. + podAntiAffinity: {} + + # -- Pod affinity constraints. + podAffinity: {} + + # -- Node affinity constraints. + nodeAffinity: {} + # Admission controller configuration admissionController: diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml index a414045fa33c..e8906f2f992f 100644 --- a/config/install-latest-testing.yaml +++ b/config/install-latest-testing.yaml @@ -35511,6 +35511,7 @@ metadata: name: kyverno-cleanup-admission-reports namespace: kyverno labels: + app.kubernetes.io/component: cleanup app.kubernetes.io/instance: kyverno app.kubernetes.io/part-of: kyverno app.kubernetes.io/version: latest @@ -35558,6 +35559,7 @@ metadata: name: kyverno-cleanup-cluster-admission-reports namespace: kyverno labels: + app.kubernetes.io/component: cleanup app.kubernetes.io/instance: kyverno app.kubernetes.io/part-of: kyverno app.kubernetes.io/version: latest diff --git a/pkg/openapi/manager.go b/pkg/openapi/manager.go index 6b4bec185312..ffb22b3e0a2c 100644 --- a/pkg/openapi/manager.go +++ b/pkg/openapi/manager.go @@ -133,11 +133,20 @@ func (o *manager) ValidatePolicyMutation(policy kyvernov1.PolicyInterface) error } for kind, rules := range kindToRules { + if kind == "CustomResourceDefinition" { + continue + } newPolicy := policy.CreateDeepCopy() spec := newPolicy.GetSpec() spec.SetRules(rules) - k, _ := o.gvkToDefinitionName.Get(kind) - d, _ := o.definitions.Get(k) + k, ok := o.gvkToDefinitionName.Get(kind) + if !ok { + continue + } + d, ok := o.definitions.Get(k) + if !ok { + continue + } resource, _ := o.generateEmptyResource(d).(map[string]interface{}) if len(resource) == 0 { o.logger.V(2).Info("unable to validate resource. OpenApi definition not found", "kind", kind) diff --git a/pkg/validation/policy/validate.go b/pkg/validation/policy/validate.go index ad8a335aea0c..e6ea81d92789 100644 --- a/pkg/validation/policy/validate.go +++ b/pkg/validation/policy/validate.go @@ -1252,6 +1252,9 @@ func validateNamespaces(s *kyvernov1.Spec, path *field.Path) error { } for i, vfa := range s.ValidationFailureActionOverrides { + if !vfa.Action.IsValid() { + return fmt.Errorf("invalid action") + } patternList, nsList := wildcard.SeperateWildcards(vfa.Namespaces) if vfa.Action.Audit() { diff --git a/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/01-policy.yaml b/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/01-policy.yaml new file mode 100644 index 000000000000..82e282409a6f --- /dev/null +++ b/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/01-policy.yaml @@ -0,0 +1,42 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + annotations: + policies.kyverno.io/category: Security + policies.kyverno.io/description: 'This policy mutates any namespace-scoped Custom + Resource Definition created by the subjects in the xteam Azure AD group + and adds the label "createdByXteam: true".' + policies.kyverno.io/subject: RBAC + policies.kyverno.io/title: Mutate Namespace-Scoped CRDs for xteam aad + group + policy.reporter.kyverno.io/minimal: minimal + generation: 1 + labels: + aws.cdk.eks/prune-c8b5941ff5f4fe911c5ee96472fda3d1f9866734a7: "" + name: mutate-xteam-namespace-scoped-crds +spec: + background: false + rules: + - match: + all: + - resources: + kinds: + - CustomResourceDefinition + subjects: + - kind: Group + name: aad:9b9had99-6k66-2222-9999-8aadb888e888 + mutate: + patchStrategicMerge: + metadata: + labels: + createdByXteam: "true" + name: mutate-xteams-crd-creation + preconditions: + all: + - key: '{{request.operation}}' + operator: Equals + value: CREATE + - key: '{{ request.object.spec.scope }}' + operator: Equals + value: Namespaced + validationFailureAction: audit \ No newline at end of file diff --git a/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/README.md b/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/README.md new file mode 100644 index 000000000000..23f9175f807e --- /dev/null +++ b/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/README.md @@ -0,0 +1,11 @@ +## Description + +This test ensures the schema validation is skipped for CustomResourceDefinition. + +## Expected Behavior + +The Pod creation should be allowed. + +## Reference Issue(s) + +https://github.com/kyverno/kyverno/issues/7844 diff --git a/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/policy-assert.yaml b/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/policy-assert.yaml new file mode 100644 index 000000000000..45ad7ff3a795 --- /dev/null +++ b/test/conformance/kuttl/policy-validation/cluster-policy/schema-validation-crd/policy-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: mutate-xteam-namespace-scoped-crds +status: + conditions: + - reason: Succeeded + status: "True" + type: Ready \ No newline at end of file