From ec7ef93a86ad5eb87a38473e584d36285c696781 Mon Sep 17 00:00:00 2001
From: Ahmed AbouZaid <6760103+aabouzaid@users.noreply.github.com>
Date: Mon, 24 Jun 2024 16:41:28 +0200
Subject: [PATCH] wip

Signed-off-by: Ahmed AbouZaid <6760103+aabouzaid@users.noreply.github.com>
---
 .github/workflows/test-regression.yaml        |  59 ++++-----
 .../testsuites/vars/files/testsuite-core.yaml | 122 +++++-------------
 .../vars/files/testsuite-preflight.yaml       |   9 +-
 .../vars/files/variables-default.yaml         |   9 +-
 .../files/variables-ingress-combined.yaml     |  13 +-
 .../testsuites/vars/kustomization.yaml        |   7 +
 .../scenarios/chart-full-setup/Taskfile.yaml  |   1 -
 .../scenarios/lib/chart-upgrade-taskfile.yaml |   2 +-
 .../lib/testsuite-deploy-taskfile.yaml        |  10 +-
 test/integration/testsuites/base/job.yaml     |   7 +-
 .../testsuites/core/kustomization.yaml        |   6 -
 .../testsuites/core/patches/job.yaml          |   1 +
 .../testsuites/preflight/kustomization.yaml   |   8 --
 13 files changed, 95 insertions(+), 159 deletions(-)
 rename test/integration/testsuites/core/testsuites/core.yaml => charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-core.yaml (72%)
 rename test/integration/testsuites/preflight/testsuites/preflight.yaml => charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-preflight.yaml (95%)

diff --git a/.github/workflows/test-regression.yaml b/.github/workflows/test-regression.yaml
index d9976a7301..6471346f11 100644
--- a/.github/workflows/test-regression.yaml
+++ b/.github/workflows/test-regression.yaml
@@ -35,38 +35,39 @@ jobs:
       with:
         charts-path: "charts/camunda-platform-8*"
 
-  validation:
-    if: ${{ needs.init.outputs.matrix != '[]' }}
-    name: Validation - Camunda ${{ matrix.version }}
-    needs: init
-    strategy:
-      fail-fast: false
-      matrix:
-        version: ${{ fromJson(needs.init.outputs.matrix) }}
-    uses: ./.github/workflows/chart-validate-template.yaml
-    with:
-      identifier: "${{ github.event.pull_request.number }}-vald-${{ matrix.version }}"
-      camunda-helm-dir: "camunda-platform-${{ matrix.version }}"
-      camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}"
+  # validation:
+  #   if: ${{ needs.init.outputs.matrix != '[]' }}
+  #   name: Validation - Camunda ${{ matrix.version }}
+  #   needs: init
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       version: ${{ fromJson(needs.init.outputs.matrix) }}
+  #   uses: ./.github/workflows/chart-validate-template.yaml
+  #   with:
+  #     identifier: "${{ github.event.pull_request.number }}-vald-${{ matrix.version }}"
+  #     camunda-helm-dir: "camunda-platform-${{ matrix.version }}"
+  #     camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}"
 
-  unit:
-    if: ${{ needs.init.outputs.matrix != '[]' }}
-    name: Unit Test - Camunda ${{ matrix.version }}
-    needs: [init]
-    strategy:
-      fail-fast: false
-      matrix:
-        version: ${{ fromJson(needs.init.outputs.matrix) }}
-    uses: ./.github/workflows/test-unit-template.yml
-    with:
-      identifier: "${{ github.event.pull_request.number }}-unit-${{ matrix.version }}"
-      camunda-helm-dir: "camunda-platform-${{ matrix.version }}"
-      camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}"
+  # unit:
+  #   if: ${{ needs.init.outputs.matrix != '[]' }}
+  #   name: Unit Test - Camunda ${{ matrix.version }}
+  #   needs: [init]
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       version: ${{ fromJson(needs.init.outputs.matrix) }}
+  #   uses: ./.github/workflows/test-unit-template.yml
+  #   with:
+  #     identifier: "${{ github.event.pull_request.number }}-unit-${{ matrix.version }}"
+  #     camunda-helm-dir: "camunda-platform-${{ matrix.version }}"
+  #     camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}"
 
   integration:
     if: ${{ needs.init.outputs.matrix != '[]' }}
     name: Integration Test - Camunda ${{ matrix.version }}
-    needs: [init, validation, unit]
+    #needs: [init, validation, unit]
+    needs: [init]
     strategy:
       fail-fast: false
       matrix:
@@ -81,7 +82,7 @@ jobs:
       identifier: "${{ github.event.pull_request.number }}-intg-${{ matrix.version }}"
       deployment-ttl: "${{ contains(github.event.pull_request.labels.*.name, 'test-persistent') && '1w' || '' }}"
       platforms: "gke"
-      flows: "install,upgrade"
-      test-enabled: false
+      flows: "install"
+      # flows: "install,upgrade"
       camunda-helm-dir: "camunda-platform-${{ matrix.version }}"
       camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}"
diff --git a/test/integration/testsuites/core/testsuites/core.yaml b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-core.yaml
similarity index 72%
rename from test/integration/testsuites/core/testsuites/core.yaml
rename to charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-core.yaml
index 4884f29911..f512ef49be 100644
--- a/test/integration/testsuites/core/testsuites/core.yaml
+++ b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-core.yaml
@@ -1,6 +1,6 @@
 # NOTE: WebModeler has its own step because Venom doesn't support skip in loops yet.
 # https://github.com/ovh/venom/issues/651
-name: Test core functionality of Camunda
+name: Test core functionality of Camunda Platform
 
 # Vars without defaults are passed as a Venom var, e.g. "VENOM_VAR_TEST_CLIENT_SECRET".
 vars:
@@ -14,10 +14,6 @@ testcases:
 
 # https://docs.camunda.io/docs/self-managed/identity/user-guide/generating-m2m-tokens/
 - name: TEST - Generating machine-to-machine token
-  description: |
-    Test generating tokens from Keycloak for components that use client type "CONFIDENTIAL".
-    This test will not work with type "PUBLIC" because "Public client not allowed to retrieve service account".
-    https://datatracker.ietf.org/doc/html/rfc6749#section-2.1
   steps:
   - name: "{{ .value.component }}"
     type: http
@@ -64,15 +60,15 @@ testcases:
       jwt:
         from: result.bodyjson.access_token
 
-- name: TEST - Interacting with Camunda login page
+# TODO: Use Venom "web" module to test actual login. It's easy but requires "PhantomJS"
+# which is not available on Alpine, so it needs to be installed first.
+- name: TEST - Interacting with Camunda Platform login page
   steps:
   - name: "{{ .value.component }}"
     skip:
     - skiptestingress ShouldBeFalse
     type: http
     range:
-    - component: Console
-      url: "{{ .coreVars.baseURLs.console }}"
     - component: Keycloak
       url: "{{ .coreVars.baseURLs.keycloak }}"
     - component: Identity
@@ -83,6 +79,8 @@ testcases:
       url: "{{ .coreVars.baseURLs.optimize }}"
     - component: Tasklist
       url: "{{ .coreVars.baseURLs.tasklist }}"
+    - component: Connectors
+      url: "{{ .coreVars.baseURLs.connectors }}"
     - component: WebModeler
       url: "{{ .coreVars.baseURLs.webModeler }}"
     method: GET
@@ -98,36 +96,12 @@ testcases:
     - result.statuscode ShouldEqual 200
     - result.body ShouldNotContainSubstring error
 
-# Connectors Inbound doesn't have the same flow like the rest of the components.
-# So it needs different check.
-- name: TEST - Interacting with Camunda login page - Connectors
-  steps:
-  - name: "Check Inbound Connectors"
-    skip:
-    - skiptestingress ShouldBeFalse
-    type: http
-    method: GET
-    url: "{{ .coreVars.baseURLs.connectors }}"
-    retry: 3
-    delay: 15
-    info: |
-      = Request Method: {{ .value.method }}
-      = Request Body: {{ .result.request.body }}
-      = Response Body: {{ .result.body }}
-    assertions:
-    - result.statuscode ShouldEqual 200
-    # - result.bodyjson.items.items0.health.status ShouldEqual UP
-
 # TODO: Add Optimize.
-- name: TEST - Interacting with Camunda web API
+- name: TEST - Interacting with Camunda Platform web API
   steps:
   - name: "{{ .value.component }}"
     type: http
     range:
-    - component: Console
-      url: "{{ .coreVars.baseURLs.console }}/api/clusters"
-      method: GET
-      body: ''
     - component: Identity
       url: "{{ .coreVars.baseURLs.identity }}/api/users"
       method: GET
@@ -160,22 +134,21 @@ testcases:
 # Hence, this test is added in the extra tests when Ingress is enabled.
 - name: TEST - Interacting with WebModeler
   steps:
-  # TODO: Check why WM 8.3.0 cannot be accessed internally.
-  # - name: Check WebModeler internal API
-  #   skip:
-  #   - skiptestwebmodeler ShouldBeFalse
-  #   type: http
-  #   method: GET
-  #   url: "{{ .coreVars.baseURLs.webModelerRestapi }}/api/v1/info"
-  #   headers:
-  #     Content-Type: application/json
-  #     Authorization: "Bearer {{ .helperVenomToken.jwt }}"
-  #   # info: |
-  #   #   = Request Body: {{ .result.request.body }}
-  #   #   = Response Body: {{ .result.body }}
-  #   assertions:
-  #   - result.statuscode ShouldEqual 200
-  #   - result.bodyjson.version ShouldNotBeEmpty
+  - name: Check WebModeler internal API
+    skip:
+    - skiptestwebmodeler ShouldBeFalse
+    type: http
+    method: GET
+    url: "{{ .coreVars.baseURLs.webModelerRestapi }}/api/beta/info"
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ .helperVenomToken.jwt }}"
+    # info: |
+    #   = Request Body: {{ .result.request.body }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+    - result.bodyjson.version ShouldNotBeEmpty
   - name: Check WebModeler login page
     skip:
     - skiptestingress ShouldBeFalse
@@ -196,12 +169,12 @@ testcases:
 
 - name: TEST - Interacting with Zeebe Gateway
   steps:
-  - name: Check Zeebe Gateway status - gRPC
+  - name: Check Zeebe Gateway status
     type: exec
     script: |
       zbctl --clientCache=/tmp/zeebe \
           --clientId {{ .venomClientID }} --clientSecret {{ .venomClientSecret }} \
-          --authzUrl "{{ .coreVars.authURL }}" --address "{{ .coreVars.baseURLs.zeebeGatewayGRPC }}" \
+          --authzUrl "{{ .coreVars.authURL }}" --address "{{ .coreVars.baseURLs.zeebeGateway }}" \
           {{ .coreVars.zbctl.extraArgs }} \
           status
     retry: 2
@@ -214,21 +187,6 @@ testcases:
     - result.systemout ShouldContainSubstring "Leader, Healthy"
     - result.systemout ShouldNotContainSubstring Unhealthy
     - result.timeseconds ShouldBeLessThan 1
-  - name: Check Zeebe Gateway status - REST
-    type: http
-    method: GET
-    url: "{{ .coreVars.baseURLs.zeebeGatewayREST }}/v1/topology"
-    headers:
-      Content-Type: application/json
-      Authorization: "Bearer {{ .helperVenomToken.jwt }}"
-    retry: 2
-    delay: 10
-    info: |
-     = Request Body: {{ .result.request.body }}
-     = Response Body: {{ .result.body }}
-    assertions:
-    - result.statuscode ShouldEqual 200
-    - result.bodyjson ShouldContainKey brokers
 
 - name: TEST - Deploying BPMN process
   steps:
@@ -242,7 +200,7 @@ testcases:
     script: |
       zbctl --clientCache=/tmp/zeebe \
           --clientId {{ .venomClientID }} --clientSecret {{ .venomClientSecret }} \
-          --authzUrl "{{ .coreVars.authURL }}" --address "{{ .coreVars.baseURLs.zeebeGatewayGRPC }}" \
+          --authzUrl "{{ .coreVars.authURL }}" --address "{{ .coreVars.baseURLs.zeebeGateway }}" \
           {{ .coreVars.zbctl.extraArgs }} \
           deploy /mnt/fixtures/{{ .value.file }}
     # info: |
@@ -268,9 +226,10 @@ testcases:
     body: '{}'
     retry: 3
     delay: 15
-    # info: |
-    #   = Request Body: {{ .result.request.body }}
-    #   = Response Body: {{ .result.body }}
+    # TODO: Remove it after debugging Operate issue.
+    info: |
+      = Request Body: {{ .result.request.body }}
+      = Response Body: {{ .result.body }}
     assertions:
     - result.statuscode ShouldEqual 200
     # NOTE: JSON keys are lowercased automatically, the original key is "bpmnProcessId".
@@ -292,25 +251,4 @@ testcases:
     #   = Request Body: {{ .result.request.body }}
     #   = Response Body: {{ .result.body }}
     assertions:
-    - result.statuscode ShouldEqual 200
-
-# - name: TEST - Check ServiceMonitor
-#   steps:
-#   - name: Check prometheus could query containers
-#     type: http
-#     method: GET
-#     url: "http://{{ .coreVars.baseURLs.prometheus }}/api/v1/query?query=system_cpu_count%7Bnamespace%3D%22{{ .coreVars.testNamespace }}%22%7D"
-#     retry: 4
-#     delay: 15
-#     # info: |
-#     #   = Request Body: {{ .result.request.body }}
-#     #   = Response Body: {{ .result.body }}
-#     assertions:
-#     - result.body ShouldContainSubstring connectors
-#     - result.body ShouldContainSubstring identity
-#     - result.body ShouldContainSubstring operate
-#     - result.body ShouldContainSubstring optimize
-#     - result.body ShouldContainSubstring tasklist
-#     - result.body ShouldContainSubstring web-modeler-restapi
-#     - result.body ShouldContainSubstring zeebe
-#     - result.body ShouldContainSubstring zeebe-gateway
+    - result.statuscode ShouldEqual 200
\ No newline at end of file
diff --git a/test/integration/testsuites/preflight/testsuites/preflight.yaml b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-preflight.yaml
similarity index 95%
rename from test/integration/testsuites/preflight/testsuites/preflight.yaml
rename to charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-preflight.yaml
index fba4d53c84..8e8deca4d2 100644
--- a/test/integration/testsuites/preflight/testsuites/preflight.yaml
+++ b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-preflight.yaml
@@ -1,6 +1,6 @@
 # NOTE: WebModeler has its own step because Venom doesn't support skip in loops yet.
 # https://github.com/ovh/venom/issues/651
-name: Run preflight checks for Camunda
+name: Run preflight checks for Camunda Platform
 
 # Vars without defaults are passed as a Venom var, e.g. "VENOM_VAR_TEST_CLIENT_SECRET".
 vars:
@@ -30,8 +30,8 @@ testcases:
       url: "{{ .preflightVars.baseURLs.tasklist }}/actuator/health/readiness"
     - component: Connectors
       url: "{{ .preflightVars.baseURLs.connectors }}/actuator/health/readiness"
-    - component: ZeebeGateway
-      url: "{{ .preflightVars.baseURLs.zeebeGateway }}/actuator/health/readiness"
+    - component: Zeebe-Gateway
+      url: "{{ .preflightVars.baseURLs.zeebeGateway }}/health"
     method: GET
     url: "{{ .value.url }}"
     retry: 3
@@ -41,6 +41,7 @@ testcases:
     #   Response Body: {{ .result.body }}
     assertions:
     - result.statuscode ShouldEqual 200
+
   - name: "WebModeler - {{ .value.component }}"
     skip:
     - skiptestwebmodeler ShouldBeFalse
@@ -81,7 +82,7 @@ testcases:
       url: "{{ .preflightVars.baseURLs.tasklist }}/actuator/health/liveness"
     - component: Connectors
       url: "{{ .preflightVars.baseURLs.connectors }}/actuator/health/liveness"
-    - component: ZeebeGateway
+    - component: Zeebe-Gateway
       url: "{{ .preflightVars.baseURLs.zeebeGateway }}/actuator/health/liveness"
     method: GET
     url: "{{ .value.url }}"
diff --git a/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-default.yaml b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-default.yaml
index bfe3750509..825204152d 100644
--- a/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-default.yaml
+++ b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-default.yaml
@@ -1,21 +1,19 @@
 preflightVars:
   baseURLs:
-    elasticsearch: http://integration-elasticsearch:9200
-    console: http://integration-console:9100
+    elasticsearch: http://elasticsearch-master:9200
     keycloak: http://integration-keycloak
     identity: http://integration-identity:82
     optimize: http://integration-optimize
     operate: http://integration-operate
     tasklist: http://integration-tasklist
     connectors: http://integration-connectors:8080
+    zeebeGateway: http://integration-zeebe-gateway:9600
     webModelerRestapi: http://integration-web-modeler-restapi:8091
     webModelerWebapp: http://integration-web-modeler-webapp:8071
-    zeebeGateway: http://integration-zeebe-gateway:9600
 
 coreVars:
   authURL: "http://integration-keycloak/auth/realms/camunda-platform/protocol/openid-connect/token"
   baseURLs:
-    console: http://integration-console
     keycloak: http://integration-keycloak
     identity: http://integration-identity
     optimize: http://integration-optimize
@@ -23,7 +21,6 @@ coreVars:
     tasklist: http://integration-tasklist
     connectors: http://integration-connectors:8080/inbound
     webModelerRestapi: http://integration-web-modeler-restapi
-    zeebeGatewayGRPC: http://integration-zeebe-gateway:9600
-    zeebeGatewayREST: http://integration-zeebe-gateway:8080
+    zeebeGateway: integration-zeebe-gateway:26500
   zbctl:
     extraArgs: "--insecure"
diff --git a/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-ingress-combined.yaml b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-ingress-combined.yaml
index 1746027b84..5db776cc4a 100644
--- a/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-ingress-combined.yaml
+++ b/charts/camunda-platform-8.2/test/integration/testsuites/vars/files/variables-ingress-combined.yaml
@@ -1,23 +1,20 @@
 preflightVars:
   baseURLs:
-    elasticsearch: http://integration-elasticsearch:9200
-    console: http://integration-console:9100
+    elasticsearch: http://elasticsearch-master:9200
     keycloak: http://integration-keycloak
     identity: http://integration-identity:82
     optimize: http://integration-optimize/optimize
     operate: http://integration-operate/operate
     tasklist: http://integration-tasklist/tasklist
     connectors: http://integration-connectors:8080/connectors
+    zeebeGateway: http://integration-zeebe-gateway:9600
     webModelerRestapi: http://integration-web-modeler-restapi:8091
     webModelerWebapp: http://integration-web-modeler-webapp:8071
-    zeebeGateway: http://integration-zeebe-gateway:9600/zeebe
 
 coreVars:
   authURL: "https://{{ .TEST_INGRESS_HOST }}/auth/realms/camunda-platform/protocol/openid-connect/token"
-  testNamespace: "{{ .K8S_NAMESPACE }}"
   baseURLs:
-    console: "https://{{ .TEST_INGRESS_HOST }}"
-    keycloak: "https://{{ .TEST_INGRESS_HOST }}/auth/"
+    keycloak: "https://{{ .TEST_INGRESS_HOST }}/auth"
     identity: "https://{{ .TEST_INGRESS_HOST }}/identity"
     operate: "https://{{ .TEST_INGRESS_HOST }}/operate"
     optimize: "https://{{ .TEST_INGRESS_HOST }}/optimize"
@@ -25,8 +22,6 @@ coreVars:
     connectors: "https://{{ .TEST_INGRESS_HOST }}/connectors/inbound"
     webModeler: "https://{{ .TEST_INGRESS_HOST }}/modeler"
     webModelerRestapi: http://integration-web-modeler-restapi
-    zeebeGatewayGRPC: "zeebe-{{ .TEST_INGRESS_HOST }}:443"
-    zeebeGatewayREST: "https://{{ .TEST_INGRESS_HOST }}/zeebe"
-    prometheus: "prometheus-operated.kube-prometheus-stack.svc.cluster.local:9090"
+    zeebeGateway: "zeebe-{{ .TEST_INGRESS_HOST }}:443"
   zbctl:
     extraArgs: ""
diff --git a/charts/camunda-platform-8.2/test/integration/testsuites/vars/kustomization.yaml b/charts/camunda-platform-8.2/test/integration/testsuites/vars/kustomization.yaml
index ced36288a4..412b1b9334 100644
--- a/charts/camunda-platform-8.2/test/integration/testsuites/vars/kustomization.yaml
+++ b/charts/camunda-platform-8.2/test/integration/testsuites/vars/kustomization.yaml
@@ -2,6 +2,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 configMapGenerator:
+# This will be added in the CI.
+# - name: venom-tests
+#   options:
+#     disableNameSuffixHash: true
+#   files:
+#   - files/testsuite-preflight.yaml
+#   - files/testsuite-core.yaml
 - name: venom-vars
   options:
     disableNameSuffixHash: true
diff --git a/test/integration/scenarios/chart-full-setup/Taskfile.yaml b/test/integration/scenarios/chart-full-setup/Taskfile.yaml
index 30b4bbe62e..9e9c8c7c2e 100644
--- a/test/integration/scenarios/chart-full-setup/Taskfile.yaml
+++ b/test/integration/scenarios/chart-full-setup/Taskfile.yaml
@@ -73,7 +73,6 @@ tasks:
         --values {{ .TEST_VALUES_BASE_DIR }}/common/values-integration-test.yaml
         --values {{ .TEST_VALUES_BASE_DIR }}/chart-full-setup/values-integration-test-ingress.yaml
         --timeout 20m0s
-        --wait
         {{ .TEST_HELM_EXTRA_ARGS }}
 
   setup.post:
diff --git a/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml b/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml
index c18e48dd0b..1d0c5623bd 100644
--- a/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml
+++ b/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml
@@ -47,4 +47,4 @@ tasks:
         --set identityKeycloak.postgresql.auth.password=$KEYCLOAK_POSTGRESQL_SECRET \
         --set identityPostgresql.auth.password=dummy \
         --timeout 20m0s \
-        --wait {{ .TEST_HELM_EXTRA_ARGS }}
+        {{ .TEST_HELM_EXTRA_ARGS }}
diff --git a/test/integration/scenarios/lib/testsuite-deploy-taskfile.yaml b/test/integration/scenarios/lib/testsuite-deploy-taskfile.yaml
index 950bc1c453..dd40ce1cb9 100644
--- a/test/integration/scenarios/lib/testsuite-deploy-taskfile.yaml
+++ b/test/integration/scenarios/lib/testsuite-deploy-taskfile.yaml
@@ -17,7 +17,15 @@ tasks:
   default:
     cmds:
     # Add testsuites vars.
-    - kubectl kustomize {{ .chartDir }}/test/integration/testsuites/vars | kubectl apply -n $TEST_NAMESPACE -f -
+    - |
+      # Kustomize doesn't support dir with edit/patch.
+      # https://github.com/kubernetes-sigs/kustomize/issues/2803
+      (
+        cd {{ .chartDir }}/test/integration/testsuites/vars &&
+        kustomize edit add configmap venom-tests --from-file=files/testsuite-{{ .testID }}.yaml
+      )
+    - |
+      kubectl kustomize {{ .chartDir }}/test/integration/testsuites/vars | kubectl apply -n $TEST_NAMESPACE -f -
     # Add imagePullSecrets if needed.
     - cmd: |
         if [[ -n "${TEST_CREATE_DOCKER_LOGIN_SECRET}" ]]; then
diff --git a/test/integration/testsuites/base/job.yaml b/test/integration/testsuites/base/job.yaml
index 23f773ec9d..731faa4e95 100644
--- a/test/integration/testsuites/base/job.yaml
+++ b/test/integration/testsuites/base/job.yaml
@@ -85,9 +85,12 @@ spec:
       volumes:
       - name: tools
         emptyDir: {}
-      - name: venom-tests
+      - name: venom-tests-preflight
         configMap:
-          name: venom-tests
+          name: venom-tests-preflight
+      - name: venom-tests-core
+        configMap:
+          name: venom-tests-core
       - name: venom-vars
         configMap:
           name: venom-vars
diff --git a/test/integration/testsuites/core/kustomization.yaml b/test/integration/testsuites/core/kustomization.yaml
index 0be8d0d650..e5c6dad1e8 100644
--- a/test/integration/testsuites/core/kustomization.yaml
+++ b/test/integration/testsuites/core/kustomization.yaml
@@ -7,12 +7,6 @@ resources:
 - ../base
 
 configMapGenerator:
-- name: venom-tests
-  options:
-    disableNameSuffixHash: true
-  behavior: merge
-  files:
-  - testsuites/core.yaml
 - name: venom-fixtures
   options:
     disableNameSuffixHash: true
diff --git a/test/integration/testsuites/core/patches/job.yaml b/test/integration/testsuites/core/patches/job.yaml
index 90e3825c70..c85786317b 100644
--- a/test/integration/testsuites/core/patches/job.yaml
+++ b/test/integration/testsuites/core/patches/job.yaml
@@ -14,6 +14,7 @@ spec:
             secretKeyRef:
               name: integration-console-identity-secret
               key: console-secret
+              optional: true
         - name: VENOM_VAR_TASKLIST_CLIENT_SECRET
           valueFrom:
             secretKeyRef:
diff --git a/test/integration/testsuites/preflight/kustomization.yaml b/test/integration/testsuites/preflight/kustomization.yaml
index 8bdca90f5a..7b009b4bba 100644
--- a/test/integration/testsuites/preflight/kustomization.yaml
+++ b/test/integration/testsuites/preflight/kustomization.yaml
@@ -5,11 +5,3 @@ nameSuffix: -preflight
 
 resources:
 - ../base
-
-configMapGenerator:
-- name: venom-tests
-  options:
-    disableNameSuffixHash: true
-  behavior: merge
-  files:
-  - testsuites/preflight.yaml