chore: split testsuites per chart version (#2041)

venom `loops` and `skip` are super buggy so it's better to split the `testsuites` per chart version

Signed-off-by: Ahmed AbouZaid <[email protected]>

.github/actions/workflow-vars/action.yml

@@ -70,7 +70,7 @@ runs:
       echo "Output vars:"
 
       # Deployment identifier.
-      TEST_IDENTIFIER="${{ inputs.platform }}-${{ inputs.identifier-base }}"
+      TEST_IDENTIFIER="$(echo ${{ inputs.platform }}-${{ inputs.identifier-base }} | sed 's/\./-/g')"
       if [[ "${{ inputs.setup-flow }}" == 'upgrade' ]]; then
         TEST_IDENTIFIER="${TEST_IDENTIFIER}-upgrade"
       fi

.github/workflows/test-integration-template.yaml

@@ -116,12 +116,15 @@ jobs:
       TEST_CLUSTER_TYPE: ${{ matrix.distro.type }}
 
     steps:
+    - name: Print workflow inputs
+      run: |
+        echo "Workflow inputs:"
+        echo "${{ toJSON(inputs) }}"
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
       with:
         # This is needed to load repo GH composite actions if the workflow triggered by workflow_call.
         repository: camunda/camunda-platform-helm
         ref: ${{ inputs.camunda-helm-git-ref }}
-
     # When there is a vault-secret-mapping input given, use Vault instead of GitHub secrets
     # and populate environment variables from Vault
     - name: Import Vault secrets
@@ -135,7 +138,6 @@ jobs:
         secretId: ${{ secrets.VAULT_SECRET_ID }}
         secrets: ${{ inputs.vault-secret-mapping }}
         exportEnv: true
-
     # Used to create/delete GitHub environment.
     # NOTE: The GH app requires "administration:write" access to be able to delete the GH environment.
     - name: Generate GitHub token

.github/workflows/test-regression.yaml

@@ -82,6 +82,5 @@ jobs:
       deployment-ttl: "${{ contains(github.event.pull_request.labels.*.name, 'test-persistent') && '1w' || '' }}"
       platforms: "gke"
       flows: "install,upgrade"
-      test-enabled: false
       camunda-helm-dir: "camunda-platform-${{ matrix.version }}"
       camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}"

charts/camunda-platform-8.2/test/integration/testsuites/vars/files/testsuite-core.yaml

@@ -0,0 +1,254 @@
+# NOTE: WebModeler has its own step because Venom doesn't support skip in loops yet.
+# https://github.com/ovh/venom/issues/651
+name: Test core functionality of Camunda Platform
+
+# Vars without defaults are passed as a Venom var, e.g. "VENOM_VAR_TEST_CLIENT_SECRET".
+vars:
+  releaseName: integration
+  venomClientID: '{{ .TEST_CLIENT_ID | default "venom" }}'
+  venomClientSecret: '{{ .TEST_CLIENT_SECRET }}'
+  skipTestIngress: '{{ .SKIP_TEST_INGRESS }}'
+  skipTestWebModeler: '{{ .SKIP_TEST_WEBMODELER }}'
+
+testcases:
+
+# https://docs.camunda.io/docs/self-managed/identity/user-guide/generating-m2m-tokens/
+- name: TEST - Generating machine-to-machine token
+  steps:
+  - name: "{{ .value.component }}"
+    type: http
+    range:
+    - component: Tasklist
+      clientID: tasklist
+      clientSecret: "{{ .TASKLIST_CLIENT_SECRET }}"
+    - component: Operate
+      clientID: operate
+      clientSecret: "{{ .OPERATE_CLIENT_SECRET }}"
+    - component: Optimize
+      clientID: optimize
+      clientSecret: "{{ .OPTIMIZE_CLIENT_SECRET }}"
+    - component: Connectors
+      clientID: connectors
+      clientSecret: "{{ .CONNECTORS_CLIENT_SECRET }}"
+    method: POST
+    url: "{{ .coreVars.authURL }}"
+    headers:
+      Content-Type: application/x-www-form-urlencoded
+    body: |-
+      client_id={{ .value.clientID }}&client_secret={{ .value.clientSecret }}&grant_type=client_credentials
+    # info: |
+    #   Component: {{ .value.component }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+
+# Helper to get access token for Venom Identity client which's seeded during Helm deployment.
+- name: helperVenomToken
+  steps:
+  - name: Get Venom token
+    type: http
+    method: POST
+    url: "{{ .coreVars.authURL }}"
+    headers:
+      Content-Type: application/x-www-form-urlencoded
+    body: "client_id={{ .venomClientID }}&client_secret={{ .venomClientSecret }}&grant_type=client_credentials"
+    # info: |
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+    vars:
+      jwt:
+        from: result.bodyjson.access_token
+
+# TODO: Use Venom "web" module to test actual login. It's easy but requires "PhantomJS"
+# which is not available on Alpine, so it needs to be installed first.
+- name: TEST - Interacting with Camunda login page
+  steps:
+  - name: "{{ .value.component }}"
+    skip:
+    - skiptestingress ShouldBeFalse
+    type: http
+    range:
+    - component: Keycloak
+      url: "{{ .coreVars.baseURLs.keycloak }}"
+    - component: Identity
+      url: "{{ .coreVars.baseURLs.identity }}"
+    - component: Operate
+      url: "{{ .coreVars.baseURLs.operate }}"
+    - component: Optimize
+      url: "{{ .coreVars.baseURLs.optimize }}"
+    - component: Tasklist
+      url: "{{ .coreVars.baseURLs.tasklist }}"
+    - component: Connectors
+      url: "{{ .coreVars.baseURLs.connectors }}"
+    - component: WebModeler
+      url: "{{ .coreVars.baseURLs.webModeler }}"
+    method: GET
+    url: "{{ .value.url }}"
+    retry: 3
+    delay: 15
+    # info: |
+    #   Component: {{ .value.component }}
+    #   = Request Method: {{ .value.method }}
+    #   = Request Body: {{ .result.request.body }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+    - result.body ShouldNotContainSubstring error
+
+# TODO: Add Optimize.
+- name: TEST - Interacting with Camunda web API
+  steps:
+  - name: "{{ .value.component }}"
+    type: http
+    range:
+    - component: Identity
+      url: "{{ .coreVars.baseURLs.identity }}/api/users"
+      method: GET
+      body: ''
+    - component: Operate
+      docs: https://docs.camunda.io/docs/self-managed/operate-deployment/operate-authentication/
+      url: "{{ .coreVars.baseURLs.operate }}/v1/process-definitions/search"
+      method: POST
+      body: '{}'
+    - component: Tasklist
+      docs: https://docs.camunda.io/docs/self-managed/tasklist-deployment/tasklist-authentication/
+      url: "{{ .coreVars.baseURLs.tasklist }}/graphql"
+      method: POST
+      body: '{"query": "{tasks(query:{}){id name}}"}'
+    method: "{{ .value.method }}"
+    url: "{{ .value.url }}"
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ .helperVenomToken.jwt }}"
+    body: "{{ .value.body }}"
+    # info: |
+    #   Component: {{ .value.component }}
+    #   = Request Method: {{ .value.method }}
+    #   = Request Body: {{ .result.request.body }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+
+# Unlike other apps, WebModeler by default expects the JWT token to be acquired from external Keycloak URL.
+# Hence, this test is added in the extra tests when Ingress is enabled.
+- name: TEST - Interacting with WebModeler
+  steps:
+  - name: Check WebModeler internal API
+    skip:
+    - skiptestwebmodeler ShouldBeFalse
+    type: http
+    method: GET
+    url: "{{ .coreVars.baseURLs.webModelerRestapi }}/api/beta/info"
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ .helperVenomToken.jwt }}"
+    # info: |
+    #   = Request Body: {{ .result.request.body }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+    - result.bodyjson.version ShouldNotBeEmpty
+  - name: Check WebModeler login page
+    skip:
+    - skiptestingress ShouldBeFalse
+    - skiptestwebmodeler ShouldBeFalse
+    type: http
+    method: GET
+    url: "{{ .coreVars.baseURLs.webModeler }}"
+    retry: 3
+    delay: 15
+    # info: |
+    #   Component: {{ .value.component }}
+    #   = Request Method: {{ .value.method }}
+    #   = Request Body: {{ .result.request.body }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+    - result.body ShouldNotContainSubstring error
+
+- name: TEST - Interacting with Zeebe Gateway
+  steps:
+  - name: Check Zeebe Gateway status
+    type: exec
+    script: |
+      zbctl --clientCache=/tmp/zeebe \
+          --clientId {{ .venomClientID }} --clientSecret {{ .venomClientSecret }} \
+          --authzUrl "{{ .coreVars.authURL }}" --address "{{ .coreVars.baseURLs.zeebeGateway }}" \
+          {{ .coreVars.zbctl.extraArgs }} \
+          status
+    retry: 2
+    delay: 10
+    # info: |
+    #   = systemerr: {{ .result.systemerr }}
+    #   = err: {{ .result.err }}
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring "Leader, Healthy"
+    - result.systemout ShouldNotContainSubstring Unhealthy
+    - result.timeseconds ShouldBeLessThan 1
+
+- name: TEST - Deploying BPMN process
+  steps:
+  - name: Deploy BPMN process to Zeebe - {{ .value.name }}
+    type: exec
+    range:
+    - name: Basic
+      file: test-process.bpmn
+    - name: Inbound Connector
+      file: test-inbound-process.bpmn
+    script: |
+      zbctl --clientCache=/tmp/zeebe \
+          --clientId {{ .venomClientID }} --clientSecret {{ .venomClientSecret }} \
+          --authzUrl "{{ .coreVars.authURL }}" --address "{{ .coreVars.baseURLs.zeebeGateway }}" \
+          {{ .coreVars.zbctl.extraArgs }} \
+          deploy /mnt/fixtures/{{ .value.file }}
+    # info: |
+    #   = systemerr: {{ .result.systemerr }}
+    #   = err: {{ .result.err }}
+    assertions:
+    - result.code ShouldEqual 0
+
+- name: TEST - Check deployed BPMN process
+  steps:
+  - name: Check deployed BPMN process in Operate - {{ .value.name }}
+    type: http
+    range:
+    - name: Basic
+      id: it-test-process
+    - name: Inbound Connector
+      id: test-inbound-process
+    method: POST
+    url: "{{ .coreVars.baseURLs.operate }}/v1/process-definitions/search"
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ .helperVenomToken.jwt }}"
+    body: '{}'
+    retry: 3
+    delay: 15
+    # TODO: Remove it after debugging Operate issue.
+    info: |
+      = Request Body: {{ .result.request.body }}
+      = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200
+    # NOTE: JSON keys are lowercased automatically, the original key is "bpmnProcessId".
+    - result.bodyjson.items.items{{ .index }}.bpmnprocessid ShouldEqual {{ .value.id }}
+
+- name: TEST - Check Connectors webhook
+  steps:
+  - name: Check deployed BPMN webhook
+    type: http
+    method: POST
+    url: "{{ .coreVars.baseURLs.connectors }}/test-mywebhook"
+    headers:
+      Content-Type: application/json
+      Authorization: "Bearer {{ .helperVenomToken.jwt }}"
+    body: '{"webhookDataKey":"webhookDataValue"}'
+    retry: 4
+    delay: 15
+    # info: |
+    #   = Request Body: {{ .result.request.body }}
+    #   = Response Body: {{ .result.body }}
+    assertions:
+    - result.statuscode ShouldEqual 200