Refactor serverless workflow (#2894)

* update scheduler * refactor agentless integration install * update tests infra * update export plugin requirement * add project init wait * update script * update script * update script call * update script * update tests marker * update wf * remove message * update tests marker * add destroy job and fix linter * update workflow * update tf format * update serverless wf * update cis-agent-based action * update config env var (cherry picked from commit 2b7683d)
elastic · Jan 13, 2025 · 9ba246e · 9ba246e
1 parent b2505a5
commit 9ba246e
Show file tree

Hide file tree

Showing 9 changed files with 155 additions and 30 deletions.
diff --git a/.github/actions/cis/action.yml → .github/actions/cis-agent-based/action.yml b/.github/actions/cis/action.yml → .github/actions/cis-agent-based/action.yml
@@ -1,5 +1,5 @@
-name: 'CIS Integrations Installation'
-description: 'Deploy CIS Integrations to Elastic Cloud'
+name: 'CIS Agent-Based Integrations Installation'
+description: 'Deploy CIS agent-based integrations to Elastic Cloud'
 inputs:
   deployment-name:
     description: |
@@ -35,10 +35,6 @@ inputs:
     description: "S3 bucket"
     required: true
     type: string
-  test-agentless:
-    description: "Run agentless integrations"
-    type: boolean
-    default: false
   es-user:
     description: "Elasticsearch user"
     default: "elastic"
@@ -58,6 +54,10 @@ inputs:
     required: false
     description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
     type: string
+  serverless-mode:
+    required: false
+    description: "Set to true if the environment is serverless"
+    default: 'false'
   tag-project:
     description: "Optional project resource tag"
     default: "test-environments"
@@ -190,7 +190,7 @@ runs:
 
     - name: Install D4C integration
       id: kspm-d4c
-      if: ${{ !cancelled() && steps.deploy-cis-infra.outcome  == 'success' }}
+      if: ${{ !cancelled() && startsWith(env.STACK_VERSION, '8') && inputs.serverless_mode == 'false' && steps.deploy-cis-infra.outcome  == 'success' }}
       working-directory: tests/integrations_setup
       shell: bash
       env:
@@ -221,11 +221,16 @@ runs:
         DEPLOYMENT_NAME: "${{ inputs.deployment-name }}"
         S3_BUCKET: "${{ inputs.env-s3-bucket }}"
         AWS_REGION: "${{ inputs.aws-region }}"
+        SERVERLESS: "${{ inputs.serverless-mode }}"
       run: |
         aws eks --region ${AWS_REGION} update-kubeconfig --name ${DEPLOYMENT_NAME} --alias eks-config
         echo 'KUBE_CONFIG_DATA=$(cat ~/.kube/config | base64)' >> $GITHUB_ENV
         kubectl config use-context eks-config
-        kubectl apply -f tests/integrations_setup/kspm_d4c.yaml
+        manifest_yaml=kspm_eks.yaml
+        if [[ "$STACK_VERSION" == 8.* && "$SERVERLESS" == "false" ]]; then
+          manifest_yaml=kspm_d4c.yaml
+        fi
+        kubectl apply -f "tests/integrations_setup/$manifest_yaml"
 
     - name: Install KSPM Unmanaged integration
       id: kspm-unmanaged
@@ -278,19 +283,6 @@ runs:
         cmd="chmod +x $scriptname && ./$scriptname"
         ../remote_setup.sh -k "$EC2_CSPM_KEY" -s "$src" -h "$CSPM_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
 
-    - name: Install Agentless integrations
-      id: agentless
-      if: ${{ !cancelled() && inputs.test-agentless == 'true' }}
-      working-directory: tests/integrations_setup
-      shell: bash
-      env:
-        AZURE_CREDENTIALS: ${{ inputs.cspm-azure-creds }}
-        ES_USER: ${{ inputs.es-user }}
-        ES_PASSWORD: ${{ inputs.es-password }}
-        KIBANA_URL: ${{ inputs.kibana-url }}
-      run: |
-        poetry run python ./install_agentless_integrations.py
-
     - name: Upload tf state
       id: upload-state-cis
       if: always()

diff --git a/.github/actions/cis-agentless/action.yml b/.github/actions/cis-agentless/action.yml
@@ -0,0 +1,37 @@
+name: 'CIS Agentless Integrations Installation'
+description: 'Deploy CIS Agentless Integrations to Elastic Cloud'
+inputs:
+  cspm-azure-creds:
+    description: "Azure credentials for CSPM agent deployment"
+    required: true
+    type: string
+  es-user:
+    description: "Elasticsearch user"
+    default: "elastic"
+    required: false
+    type: string
+  es-password:
+    description: "Elasticsearch password"
+    default: "changeme"
+    required: false
+    type: string
+  kibana-url:
+    description: "Kibana URL"
+    default: "default"
+    required: false
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: Install Agentless integrations
+      id: agentless-integrations
+      working-directory: tests/integrations_setup
+      shell: bash
+      env:
+        AZURE_CREDENTIALS: ${{ inputs.cspm-azure-creds }}
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_agentless_integrations.py
diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
@@ -103,6 +103,16 @@ on:
         type: string
         required: false
         default: "cis"
+      agent-based:
+        description: "Run agent-based integrations"
+        type: boolean
+        required: false
+        default: true
+      agentless:
+        description: "Run agentless integrations"
+        type: boolean
+        required: false
+        default: true
     outputs:
       s3-bucket:
         description: "Terraform state s3 bucket folder"
@@ -134,7 +144,6 @@ jobs:
       TF_VAR_ess_region: ${{ inputs.ess-region }}
       DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
       TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
-      TEST_AGENTLESS: true
       S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
       S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra"
       DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
@@ -233,6 +242,28 @@ jobs:
             echo "INFRA_TYPE=$INPUT_INFRA_TYPE" >> $GITHUB_ENV
           fi
 
+      - name: Init Agent Based
+        id: init-agent-based
+        env:
+          INPUT_AGENT_BASED: ${{ inputs.agent-based }}
+        run: |
+          agent_base_flag=true
+          if [[ -n "${INPUT_AGENT_BASED}" ]]; then
+            agent_base_flag=$INPUT_AGENT_BASED
+          fi
+          echo "AGENT_BASED=$agent_base_flag" >> $GITHUB_ENV
+
+      - name: Init Agentless
+        id: init-agentless
+        env:
+          INPUT_AGENTLESS: ${{ inputs.agentless }}
+        run: |
+          agentless_flag=true
+          if [[ -n "${INPUT_AGENTLESS}" ]]; then
+            agentless_flag=$INPUT_AGENTLESS
+          fi
+          echo "AGENTLESS=$agentless_flag" >> $GITHUB_ENV
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -352,10 +383,10 @@ jobs:
           tag-project: ${{ github.actor }}
           tag-owner: ${{ github.actor }}
 
-      - name: Deploy CIS Integrations
+      - name: Deploy CIS Agent Based Integrations
         id: cis-integrations
-        if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
-        uses: ./.github/actions/cis
+        if: ${{ !cancelled() && env.AGENT_BASED == 'true' && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
+        uses: ./.github/actions/cis-agent-based
         with:
           deployment-name: ${{ env.DEPLOYMENT_NAME }}
           cnvm-stack-name: ${{ env.CNVM_STACK_NAME }}
@@ -367,10 +398,20 @@ jobs:
           es-user: ${{ steps.elk-stack.outputs.es-user }}
           es-password: ${{ steps.elk-stack.outputs.es-password }}
           kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
-          test-agentless: ${{ env.TEST_AGENTLESS }}
+          serverless-mode: "${{ env.TF_VAR_serverless_mode }}"
           tag-project: ${{ github.actor }}
           tag-owner: ${{ github.actor }}
 
+      - name: Deploy CIS Agentless Integrations
+        id: cis-agentless-integrations
+        if: ${{ !cancelled() && env.AGENTLESS == 'true' && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
+        uses: ./.github/actions/cis-agentless
+        with:
+          cspm-azure-creds: ${{ secrets.AZURE_CREDENTIALS }}
+          es-user: ${{ steps.elk-stack.outputs.es-user }}
+          es-password: ${{ steps.elk-stack.outputs.es-password }}
+          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
+
       - name: Wait for agents to enroll
         id: wait-for-agents
         working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }}
@@ -380,8 +421,14 @@ jobs:
       - name: Run Sanity checks
         if: ${{ success() && inputs.run-sanity-tests == true && env.INFRA_TYPE != 'cdr' }}
         working-directory: ./tests
+        env:
+          USE_K8S: "false"
         run: |
-          poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
+          test_marker="sanity"
+          if [[ "${AGENT_BASED}" == "false" ]]; then
+            test_marker="agentless"
+          fi
+          poetry run pytest -m "$test_marker" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
 
       - name: Run UI Sanity checks (Kibana)
         uses: ./.github/actions/kibana-ftr

diff --git a/.github/workflows/weekly-serverless.yml b/.github/workflows/weekly-serverless.yml
@@ -4,7 +4,7 @@ run-name: Creating Serverless Environment by @${{ github.actor }}
 on:
     workflow_dispatch: # TODO: remove
     schedule:
-      - cron: '0 0 * * 1' # every Monday at 00:00
+      - cron: '0 2 * * *' # every day at 02:00
 
 jobs:
   naming:
@@ -15,7 +15,7 @@ jobs:
       - name: Set deployment name
         id: set_deployment_name
         run: |
-          date_name=$(echo "weekly-env-$(date +'%d-%b')" | tr '[:upper:]' '[:lower:]')
+          date_name=$(echo "prd-env-$(date +'%d%b%H%M')" | tr '[:upper:]' '[:lower:]')
           echo "date-name=$date_name" >> $GITHUB_OUTPUT
 
   deploy:
@@ -34,3 +34,15 @@ jobs:
       serverless_mode: true
       run-sanity-tests: true
       expiration_days: 0
+
+  destroy_environment:
+    needs: ["naming", "deploy"]
+    uses: ./.github/workflows/destroy-environment.yml
+    secrets: inherit
+    # Required for the 'Destroy' job in the 'destroy-environment.yml'
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    if: success()
+    with:
+      prefix: ${{ needs.naming.outputs.deployment_name }}
diff --git a/deploy/test-environments/elk-stack/wait_for_project.sh b/deploy/test-environments/elk-stack/wait_for_project.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+echo "Waiting for project to be available..."
+sleep_timeout=15
+for i in {1..20}; do
+    response=$(curl -s -H "Content-type: application/json" -H "Authorization: ApiKey ${API_KEY}" "${EC_URL}/api/v1/serverless/projects/security/${PROJECT_ID}/status")
+    phase=$(echo "$response" | jq -r '.phase')
+    if [ "$phase" = "initialized" ]; then
+        echo "Project is available!"
+        exit 0
+    else
+        echo "Iteration $i: Project phase is '$phase'. Waiting..."
+        sleep $sleep_timeout
+    fi
+done
+echo "Project is not available after retries."
+exit 1
diff --git a/deploy/test-environments/modules/serverless/main.tf b/deploy/test-environments/modules/serverless/main.tf
@@ -19,3 +19,18 @@ data "http" "project_credentials" {
   method          = "POST"
   request_headers = local.ec_headers
 }
+
+resource "null_resource" "wait_for_project" {
+  depends_on = [restapi_object.ec_project]
+
+  provisioner "local-exec" {
+    # command = local.wait_script
+    command     = "./wait_for_project.sh"
+    interpreter = ["/bin/bash", "-c"]
+    environment = {
+      "API_KEY"    = var.ec_apikey
+      "EC_URL"     = var.ec_url
+      "PROJECT_ID" = restapi_object.ec_project.api_data.id
+    }
+  }
+}
diff --git a/tests/configuration.py b/tests/configuration.py
@@ -22,7 +22,7 @@
 agent.aws_findings_timeout = 10
 agent.azure_findings_timeout = 10
 agent.cluster_type = os.getenv("CLUSTER_TYPE", "eks")  # options: vanilla / eks / vanilla_aws
-agent.agentless = os.getenv("TEST_AGENTLESS", "false") == "true"
+agent.agentless = os.getenv("AGENTLESS", "false") == "true"
 
 # The K8S Node on which the test Pod is running.
 agent.node_name = os.getenv("NODE_NAME")

diff --git a/tests/integration/tests/test_sanity_checks.py b/tests/integration/tests/test_sanity_checks.py
@@ -155,6 +155,7 @@ def test_kspm_e_k_s_findings(kspm_client, match_type):
 
 
 @pytest.mark.sanity
+@pytest.mark.agentless
 @pytest.mark.parametrize("match_type", tests_data["cis_aws"])
 def test_cspm_aws_findings(
     cspm_client,
@@ -217,6 +218,7 @@ def test_cnvm_findings(cnvm_client, match_type):
 
 
 @pytest.mark.sanity
+@pytest.mark.agentless
 @pytest.mark.parametrize("match_type", tests_data["cis_gcp"])
 def test_cspm_gcp_findings(
     cspm_client,
@@ -252,6 +254,7 @@ def test_cspm_gcp_findings(
 
 
 @pytest.mark.sanity
+@pytest.mark.agentless
 @pytest.mark.parametrize("match_type", tests_data["cis_azure"])
 def test_cspm_azure_findings(
     cspm_client,

diff --git a/tests/pyproject.toml b/tests/pyproject.toml
@@ -48,6 +48,7 @@ markers = [
     "pre_merge",
     "pre_merge_agent",
     "sanity",
+    "agentless",
     # test target markers
     "k8s_file_system_rules",
     "k8s_object_psp_rules",