Merge pull request #27 from geoadmin/sync_main_to_master

Sync master with upstream main

.buildkite/hooks/pre-command

@@ -33,7 +33,6 @@ AWS_SERVICE_ACCOUNT_SECRET_PATH=kv/ci-shared/platform-ingest/aws_ingest_ci
 PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
 
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
-GITHUB_TOKEN_VAULT_PATH=kv/ci-shared/platform-ingest/github_token
 
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
 EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
@@ -101,7 +100,7 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "publish-benchmarks" ]]; then
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
-        GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
+        GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
         export GITHUB_TOKEN
     fi
 
@@ -147,12 +146,10 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
-    GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username "${GITHUB_TOKEN_VAULT_PATH}")
-    export GITHUB_USERNAME_SECRET
-    GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email "${GITHUB_TOKEN_VAULT_PATH}")
-    export GITHUB_EMAIL_SECRET
-    GITHUB_TOKEN=$(retry 5 vault kv get -field token "${GITHUB_TOKEN_VAULT_PATH}")
-    export GITHUB_TOKEN
+    GITHUB_USERNAME="elastic-vault-github-plugin-prod"
+    GITHUB_EMAIL="[email protected]"
+    GITHUB_TOKEN=$VAULT_GITHUB_TOKEN
+    export GITHUB_TOKEN GITHUB_EMAIL GITHUB_USERNAME
   fi
 fi
 

.buildkite/pipeline.schedule-daily.yml

@@ -21,23 +21,48 @@ steps:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 7.17.23-SNAPSHOT
+        STACK_VERSION: 7.17.27-SNAPSHOT
     depends_on:
       - step: "check"
         allow_failure: false
 
-  - label: "Check integrations local stacks - Stack Version v8.15"
+  - label: "Check integrations local stacks - Stack Version v8.18"
     trigger: "integrations"
     build:
       env:
         SERVERLESS: "false"
         FORCE_CHECK_ALL: "true"
-        STACK_VERSION: 8.15.0-SNAPSHOT
+        STACK_VERSION: 8.18.0-SNAPSHOT
         PUBLISH_COVERAGE_REPORTS: "true"
     depends_on:
       - step: "check"
         allow_failure: false
 
+  - label: "Check integrations local stacks - Stack Version v8.18 - LogsDB"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 8.18.0-SNAPSHOT
+        STACK_LOGSDB_ENABLED: "true"
+        PUBLISH_COVERAGE_REPORTS: "false"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
+  - label: "Check integrations local stacks - Stack Version v9.0"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 9.0.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
   - label: "Check integrations in serverless - project: Observability"
     key: "trigger-integrations-serverless-obs"
     trigger: "integrations-serverless"

.buildkite/pipeline.schedule-weekly.yml

@@ -0,0 +1,42 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: integrations-schedule-weekly
+
+env:
+  SETUP_GVM_VERSION: "v0.5.2"
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
+
+# The pipeline is triggered by the scheduler every week
+steps:
+  - label: ":white_check_mark: Check go sources"
+    key: "check"
+    command: ".buildkite/scripts/check_sources.sh"
+    agents:
+      image: "${LINUX_AGENT_IMAGE}"
+      cpu: "8"
+      memory: "4G"
+
+  - label: "Check integrations local stacks and Elastic Agent Ubuntu docker - Stack Version v8.18"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 8.18.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false
+
+  - label: "Check integrations local stacks and non-wolfi images for Elastic Agent - Stack Version v9.0"
+    trigger: "integrations"
+    build:
+      env:
+        SERVERLESS: "false"
+        FORCE_CHECK_ALL: "true"
+        STACK_VERSION: 9.0.0-SNAPSHOT
+        PUBLISH_COVERAGE_REPORTS: "false"
+        ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "true"
+    depends_on:
+      - step: "check"
+        allow_failure: false

.buildkite/pipeline.serverless.yml

@@ -6,7 +6,7 @@ env:
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "false" # not required to set since system tests are not running yet
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.30.0'
+  K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   GH_CLI_VERSION: "2.29.0"
@@ -26,6 +26,10 @@ env:
   ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
   # Set maximum number of parallel tests to run if package allows it
   ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
+  # Enable/Disable the usage of wolfi images for Elastic Agent
+  ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
+  # Disable checking for newer versions
+  ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
 
 steps:
   - input: "Input values for the variables"
@@ -76,8 +80,10 @@ steps:
 
   - label: ":junit: Junit annotate"
     plugins:
-      - junit-annotate#v2.4.1:
+      - junit-annotate#v2.5.0:
           artifacts: "build/test-results/*.xml"
+          failed-download-exit-code: 0 # Not fail the build in case there are no XML files
+          report-skipped: true
     agents:
       provider: "gcp"  # junit plugin requires docker
 

.buildkite/pipeline.yml

@@ -1,14 +1,16 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 env:
   SETUP_GVM_VERSION: "v0.5.2"
-  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   DOCKER_COMPOSE_VERSION: "v2.24.1"
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
-  K8S_VERSION: 'v1.30.0'
+  K8S_VERSION: 'v1.31.0'
   YQ_VERSION: 'v4.35.2'
   JQ_VERSION: '1.7'
   GH_CLI_VERSION: "2.29.0"
+
+  # Agent images used in pipeline steps
+  LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
 
   # Elastic package settings
@@ -24,6 +26,10 @@ env:
   ELASTIC_PACKAGE_TEST_ENABLE_INDEPENDENT_AGENT: "true"
   # Set maximum number of parallel tests to run if package allows it
   ELASTIC_PACKAGE_MAXIMUM_NUMBER_PARALLEL_TESTS: "5"
+  # Enable/Disable the usage of wolfi images for Elastic Agent
+  ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI: "${ELASTIC_PACKAGE_DISABLE_ELASTIC_AGENT_WOLFI:-false}"
+  # Disable checking for newer versions
+  ELASTIC_PACKAGE_CHECK_UPDATE_DISABLED: "true"
 
 steps:
   - label: "Get reference from target branch"
@@ -80,9 +86,11 @@ steps:
 
   - label: ":junit: Junit annotate"
     plugins:
-      - junit-annotate#v2.4.1:
+      - junit-annotate#v2.5.0:
           artifacts: "build/test-results/*.xml"
           failed-download-exit-code: 0 # Not fail the build in case there are no XML files
+          report-skipped: true
+          always-annotate: true
     agents:
       provider: "gcp"  # junit plugin requires docker
 
@@ -96,5 +104,8 @@ steps:
     # not fail build if this step fails
     soft_fail: true
     # run this step when if it is triggered by the daily job
-    if: >
-      build.source == "trigger_job" &&  build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily"
+    # skipped for now notification for tests related to STACK_VERSION: 9
+    if: |
+      build.source == "trigger_job" &&
+      build.env('BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG') == "integrations-schedule-daily" &&
+      build.env('STACK_VERSION') !~ /^9/

.buildkite/pull-requests.json

@@ -5,7 +5,7 @@
       "pipelineSlug": "integrations",
       "allow_org_users": true,
       "allowed_repo_permissions": ["admin", "write"],
-      "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+      "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]", "github-actions[bot]"],
       "set_commit_status": true,
       "build_on_commit": true,
       "build_on_comment": true,

.buildkite/scripts/backport_branch.sh

@@ -17,7 +17,7 @@ DRY_RUN="$(buildkite-agent meta-data get DRY_RUN --default ${DRY_RUN:-"true"})"
 BASE_COMMIT="$(buildkite-agent meta-data get BASE_COMMIT --default ${BASE_COMMIT:-""})"
 PACKAGE_NAME="$(buildkite-agent meta-data get PACKAGE_NAME --default ${PACKAGE_NAME:-""})"
 PACKAGE_VERSION="$(buildkite-agent meta-data get PACKAGE_VERSION --default ${PACKAGE_VERSION:-""})"
-REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get PREMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
+REMOVE_OTHER_PACKAGES="$(buildkite-agent meta-data get REMOVE_OTHER_PACKAGES --default ${REMOVE_OTHER_PACKAGES:-"false"})"
 
 if [[ -z "$PACKAGE_NAME" ]] || [[ -z "$PACKAGE_VERSION" ]]; then
   buildkite-agent annotate "The variables **PACKAGE_NAME** or **PACKAGE_VERSION** aren't defined, please try again" --style "warning"
@@ -106,10 +106,9 @@ removeOtherPackages() {
 
 update_git_config() {
     pushd $WORKSPACE > /dev/null
-    git config --global user.name "${GITHUB_USERNAME_SECRET}"
-    git config --global user.email "${GITHUB_EMAIL_SECRET}"
+    git config --global user.name "${GITHUB_USERNAME}"
+    git config --global user.email "${GITHUB_EMAIL}"
 
-    git config remote.origin.url "https://${GITHUB_USERNAME_SECRET}:${GITHUB_TOKEN}@github.com/elastic/integrations.git"
     popd > /dev/null
 }
 

.buildkite/scripts/build_packages.sh

@@ -43,7 +43,7 @@ report_build_failure() {
 
     # if running in Buildkite , add an annotation
     if [ -n "${BUILDKITE_BRANCH+x}" ]; then
-        buildkite-agent annotate "Build package ${package} failed, not published." --ctx "ctx-build-${package}" --style "warning"
+        buildkite-agent annotate "Build package ${package} failed, not published." --context "ctx-build-${package}" --style "warning"
     fi
 }
 

.buildkite/scripts/common.sh

@@ -126,8 +126,6 @@ with_mage() {
 
     local install_packages=(
             "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
             "github.com/jstemmer/go-junit-report"
             "gotest.tools/gotestsum"
     )
@@ -464,12 +462,19 @@ is_supported_stack() {
     if [ "${kibana_version}" == "null" ]; then
         return 0
     fi
-    if [[ ! "${kibana_version}" =~ \^7\. && "${STACK_VERSION}" =~ ^7\. ]]; then
+    if [[ ( ! "${kibana_version}" =~ \^7\. ) && "${STACK_VERSION}" =~ ^7\. ]]; then
         return 1
     fi
-    if [[ ! "${kibana_version}" =~ \^8\. && "${STACK_VERSION}" =~ ^8\. ]]; then
+    if [[ ( ! "${kibana_version}" =~ \^8\. ) && "${STACK_VERSION}" =~ ^8\. ]]; then
         return 1
     fi
+
+    # TODO: Allowed temporarily to test packages with stack version 9.0 if they have as constraint ^8.0 defined too.
+    # This workaround should be removed once packages have been updated their constraints for 9.0 stack.
+    if [[ ( ! ( "${kibana_version}" =~ \^9\. || "${kibana_version}" =~ \^8\. ) ) && "${STACK_VERSION}" =~ ^9\. ]]; then
+        return 1
+    fi
+
     return 0
 }
 
@@ -503,6 +508,10 @@ prepare_stack() {
         fi
     fi
 
+    if [ "${STACK_LOGSDB_ENABLED:-false}" == "true" ]; then
+        args="${args} -U stack.logsdb_enabled=true"
+    fi
+
     echo "Boot up the Elastic stack"
     if ! ${ELASTIC_PACKAGE_BIN} stack up -d ${args} ; then
         return 1
@@ -692,7 +701,7 @@ is_pr_affected() {
 
     echo "[${package}] git-diff: check non-package files"
     commit_merge=$(git merge-base "${from}" "${to}")
-    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|.github/CODEOWNERS|README.md|docs/)' ; then
+    if git diff --name-only "${commit_merge}" "${to}" | grep -E -v '^(packages/|\.github/(CODEOWNERS|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)|README\.md|docs/)' ; then
         echo "[${package}] PR is affected: found non-package files"
         return 0
     fi