Merge branch 'main' into admin-stats-histograms

Signed-off-by: Joshua Marantz <[email protected]>

.azure-pipelines/bazel.yml → .azure-pipelines/ci.yml

@@ -176,31 +176,68 @@ steps:
     tmpfsDockerDisabled: "${{ parameters.tmpfsDockerDisabled }}"
 
 - script: |
-    if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then
-        unset GOOGLE_BES_PROJECT_ID
+    ENVOY_SHARED_TMP_DIR=/tmp/bazel-shared
+    mkdir -p "$ENVOY_SHARED_TMP_DIR"
+    BAZEL_BUILD_EXTRA_OPTIONS="${{ parameters.bazelBuildExtraOptions }}"
+    if [[ "${{ parameters.rbe }}" == "True" ]]; then
+        # mktemp will create a tempfile with u+rw permission minus umask, it will not be readable by all
+        # users by default.
+        GCP_SERVICE_ACCOUNT_KEY_PATH=$(mktemp -p "${ENVOY_SHARED_TMP_DIR}" -t gcp_service_account.XXXXXX.json)
+        bash -c 'echo "$(GcpServiceAccountKey)"' | base64 --decode > "${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+        BAZEL_BUILD_EXTRA_OPTIONS+=" ${{ parameters.bazelConfigRBE }} --google_credentials=${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+        ENVOY_RBE=1
+        if [[ "${{ parameters.bazelUseBES }}" == "True" && -n "${GOOGLE_BES_PROJECT_ID}" ]]; then
+            BAZEL_BUILD_EXTRA_OPTIONS+=" --config=rbe-google-bes --bes_instance_name=${GOOGLE_BES_PROJECT_ID}"
+        fi
+    else
+        echo "using local build cache."
+        # Normalize branches - `release/vX.xx`, `vX.xx`, `vX.xx.x` -> `vX.xx`
+        TARGET_BRANCH=$(echo "${CI_TARGET_BRANCH}" | cut -d/ -f2-)
+        BRANCH_NAME="$(echo "${TARGET_BRANCH}" | cut -d/ -f2 | cut -d. -f-2)"
+        if [[ "$BRANCH_NAME" == "merge" ]]; then
+            # Manually run PR commit - there is no easy way of telling which branch
+            # it is, so just set it to `main` - otherwise it tries to cache as `branch/merge`
+            BRANCH_NAME=main
+        fi
+        BAZEL_REMOTE_INSTANCE="branch/${BRANCH_NAME}"
+        echo "instance_name: ${BAZEL_REMOTE_INSTANCE}."
+        BAZEL_BUILD_EXTRA_OPTIONS+=" --config=ci --config=cache-local --remote_instance_name=${BAZEL_REMOTE_INSTANCE} --remote_timeout=600"
     fi
-    ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}'
-  condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
+    if [[ "${{ parameters.cacheTestResults }}" != "True" ]]; then
+        VERSION_DEV="$(cut -d- -f2 "VERSION.txt")"
+        # Use uncached test results for non-release scheduledruns.
+        if [[ $VERSION_DEV == "dev" ]]; then
+            BAZEL_EXTRA_TEST_OPTIONS+=" --nocache_test_results"
+        fi
+    fi
+    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
+    if [[ "${{ variables['Build.Reason'] }}" == "PullRequest" || "${{ variables['Build.DefinitionName'] }}" == 'envoy-presubmit' ]]; then
+        # sha1sum of `ENVOY_PULL_REQUEST`
+        BAZEL_FAKE_SCM_REVISION=e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
+    fi
+    echo "##vso[task.setvariable variable=BAZEL_BUILD_EXTRA_OPTIONS]${BAZEL_BUILD_EXTRA_OPTIONS}"
+    echo "##vso[task.setvariable variable=BAZEL_EXTRA_TEST_OPTIONS]${BAZEL_EXTRA_TEST_OPTIONS}"
+    echo "##vso[task.setvariable variable=BAZEL_FAKE_SCM_REVISION]${BAZEL_FAKE_SCM_REVISION}"
+    echo "##vso[task.setvariable variable=BAZEL_STARTUP_EXTRA_OPTIONS]${{ parameters.bazelStartupExtraOptions }}"
+    echo "##vso[task.setvariable variable=CI_TARGET_BRANCH]${CI_TARGET_BRANCH}"
+    echo "##vso[task.setvariable variable=ENVOY_BUILD_FILTER_EXAMPLE]${{ parameters.envoyBuildFilterExample }}"
+    echo "##vso[task.setvariable variable=ENVOY_DOCKER_BUILD_DIR]$(Build.StagingDirectory)"
+    echo "##vso[task.setvariable variable=ENVOY_RBE]${ENVOY_RBE}"
+    echo "##vso[task.setvariable variable=ENVOY_SHARED_TMP_DIR]${ENVOY_SHARED_TMP_DIR}"
+    echo "##vso[task.setvariable variable=GCP_SERVICE_ACCOUNT_KEY_PATH]${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+    echo "##vso[task.setvariable variable=GITHUB_TOKEN]${{ parameters.authGithub }}"
   workingDirectory: $(Build.SourcesDirectory)
   env:
-    ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-    GITHUB_TOKEN: "${{ parameters.authGithub }}"
-    BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}"
     ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
       CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)"
     ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
       CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)"
-    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
-    ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}:
-      # sha1sum of `ENVOY_PULL_REQUEST`
-      BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
-    ${{ if parameters.rbe }}:
-      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-      ENVOY_RBE: "1"
-      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}"
-    ${{ if eq(parameters.rbe, false) }}:
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
-      BAZEL_REMOTE_CACHE: $(LocalBuildCache)
+  displayName: "CI env ${{ parameters.ciTarget }}"
+
+- script: ci/run_envoy_docker.sh 'ci/do_ci.sh fetch-${{ parameters.ciTarget }}'
+  condition: and(not(canceled()), not(failed()), ne('${{ parameters.cacheName }}', ''), ne(variables.CACHE_RESTORED, 'true'))
+  workingDirectory: $(Build.SourcesDirectory)
+  env:
     ${{ each var in parameters.env }}:
       ${{ var.key }}: ${{ var.value }}
   displayName: "Fetch assets (${{ parameters.ciTarget }})"
@@ -231,34 +268,10 @@ steps:
   displayName: "Enable IPv6"
   condition: ${{ parameters.managedAgent }}
 
-- script: |
-    if [[ "${{ parameters.bazelUseBES }}" == 'false' ]]; then
-        unset GOOGLE_BES_PROJECT_ID
-    fi
-    ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
+- script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}'
   workingDirectory: $(Build.SourcesDirectory)
   env:
-    ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
     ENVOY_BUILD_FILTER_EXAMPLE: ${{ parameters.envoyBuildFilterExample }}
-    GITHUB_TOKEN: "${{ parameters.authGithub }}"
-    BAZEL_STARTUP_EXTRA_OPTIONS: "${{ parameters.bazelStartupExtraOptions }}"
-    ${{ if ne(parameters['cacheTestResults'], true) }}:
-      BAZEL_NO_CACHE_TEST_RESULTS: 1
-    ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-      CI_TARGET_BRANCH: "origin/$(System.PullRequest.TargetBranch)"
-    ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-      CI_TARGET_BRANCH: "origin/$(Build.SourceBranchName)"
-    # Any PR or CI run in envoy-presubmit uses the fake SCM hash
-    ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(variables['Build.DefinitionName'], 'envoy-presubmit')) }}:
-      # sha1sum of `ENVOY_PULL_REQUEST`
-      BAZEL_FAKE_SCM_REVISION: e3b4a6e9570da15ac1caffdded17a8bebdc7dfc9
-    ${{ if parameters.rbe }}:
-      GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-      ENVOY_RBE: "1"
-      BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelConfigRBE }} ${{ parameters.bazelBuildExtraOptions }}"
-    ${{ if eq(parameters.rbe, false) }}:
-      BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci ${{ parameters.bazelBuildExtraOptions }}"
-      BAZEL_REMOTE_CACHE: $(LocalBuildCache)
     ${{ each var in parameters.env }}:
       ${{ var.key }}: ${{ var.value }}
   displayName: "Run CI script ${{ parameters.ciTarget }}"
@@ -296,6 +309,13 @@ steps:
   - ${{ each pair in step }}:
       ${{ pair.key }}: ${{ pair.value }}
 
+- bash: |
+    if [[ -n "$GCP_SERVICE_ACCOUNT_KEY_PATH" && -e "$GCP_SERVICE_ACCOUNT_KEY_PATH" ]]; then
+        echo "Removed key: ${GCP_SERVICE_ACCOUNT_KEY_PATH}"
+        rm -rf "$GCP_SERVICE_ACCOUNT_KEY_PATH"
+    fi
+  condition: not(canceled())
+
 - script: |
     set -e
     sudo .azure-pipelines/docker/save_cache.sh "$(Build.StagingDirectory)" /mnt/cache/all true true

.azure-pipelines/env.yml

@@ -135,19 +135,12 @@ jobs:
       # TODO(phlax): move this to a script to ensure proper linting etc
       set -e
 
-      # Run everything in postsubmit
-      if [[ "$(Build.Reason)" != "PullRequest" ]]; then
-          echo "##vso[task.setvariable variable=build;isoutput=true]true"
-          echo "##vso[task.setvariable variable=checks;isoutput=true]true"
-          echo "##vso[task.setvariable variable=docker;isoutput=true]true"
-          echo "##vso[task.setvariable variable=packaging;isoutput=true]true"
-          exit 0
-      fi
-
       RUN_BUILD=true
       RUN_CHECKS=true
       RUN_DOCKER=true
       RUN_PACKAGING=true
+      RUN_RELEASE_TESTS=true
+
       if [[ "$(changed.mobileOnly)" == true || "$(changed.docsOnly)" == true ]]; then
           RUN_BUILD=false
           RUN_DOCKER=false
@@ -159,10 +152,25 @@ jobs:
       if [[ "$(changed.examplesOnly)" == true ]]; then
           RUN_CHECKS=false
       fi
+      if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$(state.isDev)" == false ]]; then
+          RUN_RELEASE_TESTS=false
+      fi
+
+      # Run ~everything in postsubmit
+      if [[ "$(Build.Reason)" != "PullRequest" ]]; then
+          echo "##vso[task.setvariable variable=build;isoutput=true]true"
+          echo "##vso[task.setvariable variable=checks;isoutput=true]true"
+          echo "##vso[task.setvariable variable=docker;isoutput=true]true"
+          echo "##vso[task.setvariable variable=packaging;isoutput=true]true"
+          echo "##vso[task.setvariable variable=releaseTests;isoutput=true]${RUN_RELEASE_TESTS}"
+          exit 0
+      fi
+
       echo "##vso[task.setvariable variable=build;isoutput=true]${RUN_BUILD}"
       echo "##vso[task.setvariable variable=checks;isoutput=true]${RUN_CHECKS}"
       echo "##vso[task.setvariable variable=docker;isoutput=true]${RUN_DOCKER}"
       echo "##vso[task.setvariable variable=packaging;isoutput=true]${RUN_PACKAGING}"
+      echo "##vso[task.setvariable variable=releaseTests;isoutput=true]${RUN_RELEASE_TESTS}"
 
     displayName: "Decide what to run"
     workingDirectory: $(Build.SourcesDirectory)
@@ -174,32 +182,18 @@ jobs:
 
       PUBLISH_GITHUB_RELEASE=$(run.packaging)
       PUBLISH_DOCKERHUB=false
-      PUBLISH_DOCS=false
-      PUBLISH_DOCS_LATEST=false
-      PUBLISH_DOCS_RELEASE=false
 
       if [[ "$ISSTABLEBRANCH" == True && -n "$POSTSUBMIT" && "$NOSYNC" != true ]]; then
-          # Build docs for publishing either latest or a release build
-          PUBLISH_DOCS=true
           # main
           if [[ "$ISMAIN" == True ]]; then
               # Update the Dockerhub README
               PUBLISH_DOCKERHUB=true
-              if [[ "$(state.isDev)" == true ]]; then
-                  # Postsubmit on `main` trigger rebuild of latest docs
-                  PUBLISH_DOCS_LATEST=true
-              fi
           # Not main, and not -dev
           elif [[ "$(state.isDev)" == false ]]; then
               if [[ "$(state.versionPatch)" -eq 0 ]]; then
                   # A just-forked branch
                   PUBLISH_GITHUB_RELEASE=false
               fi
-              # A stable release, publish docs to the release
-              PUBLISH_DOCS_RELEASE=true
-          else
-              # Postsubmit for non-main/release, skip publishing docs in this case
-              PUBLISH_DOCS=false
           fi
       fi
 
@@ -210,9 +204,6 @@ jobs:
 
       echo "##vso[task.setvariable variable=githubRelease;isoutput=true]${PUBLISH_GITHUB_RELEASE}"
       echo "##vso[task.setvariable variable=dockerhub;isoutput=true]${PUBLISH_DOCKERHUB}"
-      echo "##vso[task.setvariable variable=docs;isoutput=true]${PUBLISH_DOCS}"
-      echo "##vso[task.setvariable variable=docsLatest;isoutput=true]${PUBLISH_DOCS_LATEST}"
-      echo "##vso[task.setvariable variable=docsRelease;isoutput=true]${PUBLISH_DOCS_RELEASE}"
 
     displayName: "Decide what to publish"
     workingDirectory: $(Build.SourcesDirectory)
@@ -231,12 +222,10 @@ jobs:
       echo "env.outputs['run.build']: $(run.build)"
       echo "env.outputs['run.checks']: $(run.checks)"
       echo "env.outputs['run.packaging']: $(run.packaging)"
+      echo "env.outputs['run.releaseTests']: $(run.releaseTests)"
       echo
       echo "env.outputs['publish.githubRelease']: $(publish.githubRelease)"
       echo "env.outputs['publish.dockerhub]: $(publish.dockerhub)"
-      echo "env.outputs['publish.docs]: $(publish.docs)"
-      echo "env.outputs['publish.docsLatest]: $(publish.docsLatest)"
-      echo "env.outputs['publish.docsRelease]: $(publish.docsRelease)"
 
     displayName: "Print build environment"
 

.azure-pipelines/pipelines.yml

@@ -45,7 +45,7 @@ variables:
 ## Variable settings
 # Caches (tip: append a version suffix while testing caches)
 - name: cacheKeyVersion
-  value: v2
+  value: v3
 - name: cacheKeyBazel
   value: '.bazelversion | ./WORKSPACE | **/*.bzl, !mobile/**, !envoy-docs/**'
 - name: cacheKeyDocker
@@ -80,10 +80,6 @@ stages:
       - env
       checkStageDeps:
       - env
-      macBuildStageDeps:
-      - env
-      windowsBuildStageDeps:
-      - env
 
 # Postsubmit main/release branches
 - ${{ if eq(variables.pipelinePostsubmit, true) }}:
@@ -96,7 +92,3 @@ stages:
       - env
       checkStageDeps:
       - env
-      macBuildStageDeps:
-      - env
-      windowsBuildStageDeps:
-      - env

.azure-pipelines/stage/checks.yml

@@ -77,7 +77,7 @@ jobs:
   timeoutInMinutes: 180
   pool: envoy-x64-small
   steps:
-  - template: ../bazel.yml
+  - template: ../ci.yml
     parameters:
       ciTarget: $(CI_TARGET)
       cacheName: $(CI_TARGET)
@@ -101,15 +101,7 @@ jobs:
         displayName: "Upload $(CI_TARGET) Report to GCS"
         condition: and(not(canceled()), or(eq(variables['CI_TARGET'], 'coverage'), eq(variables['CI_TARGET'], 'fuzz_coverage')))
         env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          ENVOY_RBE: "1"
-          BAZEL_BUILD_EXTRA_OPTIONS: "--config=ci --config=rbe-google --jobs=$(RbeJobs)"
-          GCP_SERVICE_ACCOUNT_KEY: ${{ parameters.authGCP }}
           GCS_ARTIFACT_BUCKET: ${{ parameters.bucketGCP }}
-          ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
-            BAZEL_REMOTE_INSTANCE_BRANCH: "$(System.PullRequest.TargetBranch)"
-          ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
-            BAZEL_REMOTE_INSTANCE_BRANCH: "$(Build.SourceBranchName)"
 
 - job: complete
   displayName: "Checks complete"

.azure-pipelines/stage/linux.yml

@@ -11,6 +11,10 @@ parameters:
   displayName: "Artifact suffix"
   type: string
   default:
+- name: runTests
+  displayName: "Run release tests"
+  type: string
+  default: true
 - name: rbe
   displayName: "Use RBE"
   type: boolean
@@ -22,6 +26,9 @@ parameters:
 - name: bazelBuildExtraOptions
   type: string
   default: ""
+- name: bazelConfigRBE
+  type: string
+  default: --config=remote-ci --config=rbe-google --jobs=$(RbeJobs)
 
 - name: managedAgent
   type: boolean
@@ -45,11 +52,21 @@ jobs:
   timeoutInMinutes: ${{ parameters.timeoutBuild }}
   pool: ${{ parameters.pool }}
   steps:
-  - template: ../bazel.yml
+  - bash: |
+      if [[ "${{ parameters.runTests }}" == "false" ]]; then
+          CI_TARGET="release.server_only"
+      else
+          CI_TARGET="release"
+      fi
+      echo "${CI_TARGET}"
+      echo "##vso[task.setvariable variable=value;isoutput=true]${CI_TARGET}"
+    name: target
+  - template: ../ci.yml
     parameters:
       managedAgent: ${{ parameters.managedAgent }}
-      ciTarget: release
+      ciTarget: $(target.value)
       cacheName: "release"
+      bazelConfigRBE: ${{ parameters.bazelConfigRBE }}
       bazelBuildExtraOptions: ${{ parameters.bazelBuildExtraOptions }}
       cacheTestResults: ${{ parameters.cacheTestResults }}
       cacheVersion: $(cacheKeyBazel)